diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 7a7a79d83..0b9aeb3d3 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -15,6 +15,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} submodules: recursive + - name: RunCiTest id: run_ci_test env: @@ -22,7 +23,7 @@ jobs: PULL_REQUEST_NUM: ${{ github.event.pull_request.number }} run: | source ~/.bashrc - source activate tf25_py3 + conda activate tf25_py3 pre-commit run -a if [ $? -eq 0 ] then diff --git a/.gitignore b/.gitignore index 227b49663..5b1c97f62 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,5 @@ pai_jobs/easy_rec*.tar.gz .DS_Store .python-version +easy_rec/python/test/odps_input_v3_test.py +easy_rec/python/test.py diff --git a/README.md b/README.md index b3f88506d..2b75da77a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # EasyRec Introduction -🎉 See our ongoing recommendation framework **[TorchEasyRec](https://github.com/alibaba/TorchEasyRec) !** 🎉 This evolution of EasyRec is built on **PyTorch**, featuring **GPU acceleration** and **hybrid parallelism** for enhanced performance. +🎉 See our ongoing recommendation framework **[TorchEasyRec](https://github.com/alibaba/TorchEasyRec) !** 🎉 This evolution of EasyRec is built on **PyTorch**, featuring **GPU acceleration** and **hybrid parallelism** for enhanced performance.   diff --git a/docs/source/automl/hpo_config.md b/docs/source/automl/hpo_config.md index 5f99e42e6..33bbb725d 100644 --- a/docs/source/automl/hpo_config.md +++ b/docs/source/automl/hpo_config.md @@ -80,7 +80,7 @@ assessor: - cmd = cmd.replace('${params}', params)->支持参数标识路径，例如lr0.001_batchsize64 注意其中可能含有浮点数，请确定是否支持用来标识数据/数据表 -- cmd = cmd.replace(p, str(v)) 将搜索的参数替换为搜索的值，搜索参数可以使用${batch_size}、${lr}来标记，需要和search_space.json中的key匹配使用 +- cmd = cmd.replace(p, str(v)) 将搜索的参数替换为搜索的值，搜索参数可以使用${batch_size}、${lr}来标记，需要和search_space.json中的key匹配使用 ### jinja渲染 @@ -224,14 +224,14 @@ metric_source=cmd1 ## odps_config -| odps_config | 描述 | 值 | -| ------------- | ------------ | -------------------------------------------------------------------------------------- | -| access_id | ak | ak | -| access_key | sk | ak | -| project_name | project_name | xxx | -| end_point | end_point | 弹外: http://service.odps.aliyun.com/api 弹内：http://service-corp.odps.aliyun-inc.com/api | -| log_view_host | logview host | 弹外：http://logview.odps.aliyun.com 弹内：http://logview.alibaba-inc.com | -| role_arn | role_arn | acs:ram::xxx:role/aliyunserviceroleforpaiautoml | +| odps_config | 描述 | 值 | +| ------------- | ------------ | ------------------------------------------------------------------------------------- | +| access_id | ak | ak | +| access_key | sk | ak | +| project_name | project_name | xxx | +| end_point | end_point | 弹外: http://service.odps.aliyun.com/api 弹内：http://service-corp.odps.aliyun-inc.com/api | +| log_view_host | logview host | 弹外：http://logview.odps.aliyun.com 弹内：http://logview.alibaba-inc.com | +| role_arn | role_arn | acs:ram::xxx:role/aliyunserviceroleforpaiautoml | ## dlc_config diff --git a/docs/source/automl/hpo_emr.md b/docs/source/automl/hpo_emr.md index 36cf76c4b..3d7172f5a 100644 --- a/docs/source/automl/hpo_emr.md +++ b/docs/source/automl/hpo_emr.md @@ -17,13 +17,13 @@ python -m easy_rec.python.hpo.emr_hpo --hyperparams hyperparams.json --config_p ### 参数说明 -- --config_path easyrec训练配置文件 -- --exp_dir 调优实验目录 -- --debug 保留本地临时目录 +- --config_path easyrec训练配置文件 +- --exp_dir 调优实验目录 +- --debug 保留本地临时目录 - --metric_name  调优的指标，默认是auc，其它可选指标\[参考../eval.md) - --max_parallel   同一时刻可以并行跑的实验数目，默认4 - --total_trial_num  总共跑多少组实验，默认6 -- --el_submit_params el_submit指定PS/Worker资源的一些参数，包括-t x -m x \[-pn x -pc x -pm x\] -wn x -wc x -wm x -wg x 默认值 +- --el_submit_params el_submit指定PS/Worker资源的一些参数，包括-t x -m x \[-pn x -pc x -pm x\] -wn x -wc x -wm x -wg x 默认值 ```bash -t standalone -m local -wn 1 -wc 6 -wm 20000 -wg 1 @@ -43,7 +43,7 @@ python -m easy_rec.python.hpo.emr_hpo --hyperparams hyperparams.json --config_p ] ``` -- name: easy_rec pipeline_config里面的参数名称，注意要用全路径 +- name: easy_rec pipeline_config里面的参数名称，注意要用全路径 feature_config.features\[**input_names\[0\]=field_name1**\].embedding_dim @@ -101,7 +101,7 @@ python -m easy_rec.python.hpo.emr_hpo --hyperparams hyperparams.json --config_p ![image.png](../../images/automl/emr_log.png) 一共做了5组实验，可以看到embedding_dim越小越好。 -- 实验目录信息(exp_dir): hdfs:///user/easy_rec_test/experiment/hpo_test_v8 +- 实验目录信息(exp_dir): hdfs:///user/easy_rec_test/experiment/hpo_test_v8 ![image.png](../../images/automl/emr_exp.png) diff --git a/docs/source/automl/hpo_pai.md b/docs/source/automl/hpo_pai.md index ac2310a41..9f547dc88 100644 --- a/docs/source/automl/hpo_pai.md +++ b/docs/source/automl/hpo_pai.md @@ -45,9 +45,9 @@ accessKeyID = xxx accessKeySecret= xxx ``` -- --bucket oss_bucket +- --bucket oss_bucket -- --role_arn acs:ram::xxx:role/xxx +- --role_arn acs:ram::xxx:role/xxx pai tensorflow 任务访问oss数据的钥匙，[获取方式](https://help.aliyun.com/document_detail/190477.html?spm=h2-url-1)。 @@ -57,13 +57,13 @@ accessKeySecret= xxx - --exp_dir 调优目录, oss上的目录 -- --config_path easyrec训练配置文件 +- --config_path easyrec训练配置文件 -- --metric_name 调优的指标，默认是auc，其它可选指标[参考](../eval.md) +- --metric_name 调优的指标，默认是auc，其它可选指标[参考](../eval.md) -- --max_parallel 同一时刻可以并行跑的实验数目 +- --max_parallel 同一时刻可以并行跑的实验数目 -- --total_trial_num 总共跑多少组实验 +- --total_trial_num 总共跑多少组实验 - --is_outer 内部pai还是外部pai @@ -89,7 +89,7 @@ accessKeySecret= xxx ] ``` -- name: easy_rec pipeline_config里面的参数名称，注意要用全路径 +- name: easy_rec pipeline_config里面的参数名称，注意要用全路径 ``` feature_config.features[input_names[0]=field1].embedding_dim ``` diff --git a/docs/source/automl/pai_nni_hpo.md b/docs/source/automl/pai_nni_hpo.md index a6e03b6b6..8f5a75fe4 100644 --- a/docs/source/automl/pai_nni_hpo.md +++ b/docs/source/automl/pai_nni_hpo.md @@ -106,8 +106,8 @@ nnictl create --config exp.yml ``` - 启动入口为exp.yml -- 通过trialCommand: python3 -m hpo_tools.core.utils.run --config=./trial.ini 连接用户的具体的启动任务。 -- 通过字段searchSpaceFile: search_space.json 连接 search_space.json; +- 通过trialCommand: python3 -m hpo_tools.core.utils.run --config=./trial.ini 连接用户的具体的启动任务。 +- 通过字段searchSpaceFile: search_space.json 连接 search_space.json; 配置案例均可以在安装目录examples/search目录下，细节请参考[HPO配置介绍](./hpo_config.md) diff --git a/docs/source/component/backbone.md b/docs/source/component/backbone.md index 0e600dbb8..195661301 100644 --- a/docs/source/component/backbone.md +++ b/docs/source/component/backbone.md @@ -1102,10 +1102,12 @@ MovieLens-1M数据集效果： 定义一个继承[`tf.keras.layers.Layer`](https://keras.io/api/layers/base_layer/)的组件类，至少实现两个方法：`__init__`、`call`。 ```python -def __init__(self, params, name='xxx', reuse=None, **kwargs): - pass +def __init__(self, params, name="xxx", reuse=None, **kwargs): + pass + + def call(self, inputs, training=None, **kwargs): - pass + pass ``` `__init__`方法的第一个参数`params`接受框架传递给当前组件的参数。支持两种参数配置的方式：`google.protobuf.Struct`、自定义的protobuf message对象。params对象封装了对这两种格式的参数的统一读取接口，如下： @@ -1143,43 +1145,45 @@ FM layer的代码示例： ```python class FM(tf.keras.layers.Layer): - """Factorization Machine models pairwise (order-2) feature interactions without linear term and bias. - - References - - [Factorization Machines](https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf) - Input shape. - - List of 2D tensor with shape: ``(batch_size,embedding_size)``. - - Or a 3D tensor with shape: ``(batch_size,field_size,embedding_size)`` - Output shape - - 2D tensor with shape: ``(batch_size, 1)``. - """ - - def __init__(self, params, name='fm', reuse=None, **kwargs): - super(FM, self).__init__(name, **kwargs) - self.reuse = reuse - self.use_variant = params.get_or_default('use_variant', False) - - def call(self, inputs, **kwargs): - if type(inputs) == list: - emb_dims = set(map(lambda x: int(x.shape[-1]), inputs)) - if len(emb_dims) != 1: - dims = ','.join([str(d) for d in emb_dims]) - raise ValueError('all embedding dim must be equal in FM layer:' + dims) - with tf.name_scope(self.name): - fea = tf.stack(inputs, axis=1) - else: - assert inputs.shape.ndims == 3, 'input of FM layer must be a 3D tensor or a list of 2D tensors' - fea = inputs - - with tf.name_scope(self.name): - square_of_sum = tf.square(tf.reduce_sum(fea, axis=1)) - sum_of_square = tf.reduce_sum(tf.square(fea), axis=1) - cross_term = tf.subtract(square_of_sum, sum_of_square) - if self.use_variant: - cross_term = 0.5 * cross_term - else: - cross_term = 0.5 * tf.reduce_sum(cross_term, axis=-1, keepdims=True) - return cross_term + """Factorization Machine models pairwise (order-2) feature interactions without linear term and bias. + + References + - [Factorization Machines](https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf) + Input shape. + - List of 2D tensor with shape: ``(batch_size,embedding_size)``. + - Or a 3D tensor with shape: ``(batch_size,field_size,embedding_size)`` + Output shape + - 2D tensor with shape: ``(batch_size, 1)``. + """ + + def __init__(self, params, name="fm", reuse=None, **kwargs): + super(FM, self).__init__(name, **kwargs) + self.reuse = reuse + self.use_variant = params.get_or_default("use_variant", False) + + def call(self, inputs, **kwargs): + if type(inputs) == list: + emb_dims = set(map(lambda x: int(x.shape[-1]), inputs)) + if len(emb_dims) != 1: + dims = ",".join([str(d) for d in emb_dims]) + raise ValueError("all embedding dim must be equal in FM layer:" + dims) + with tf.name_scope(self.name): + fea = tf.stack(inputs, axis=1) + else: + assert ( + inputs.shape.ndims == 3 + ), "input of FM layer must be a 3D tensor or a list of 2D tensors" + fea = inputs + + with tf.name_scope(self.name): + square_of_sum = tf.square(tf.reduce_sum(fea, axis=1)) + sum_of_square = tf.reduce_sum(tf.square(fea), axis=1) + cross_term = tf.subtract(square_of_sum, sum_of_square) + if self.use_variant: + cross_term = 0.5 * cross_term + else: + cross_term = 0.5 * tf.reduce_sum(cross_term, axis=-1, keepdims=True) + return cross_term ``` # 如何搭建模型 diff --git a/docs/source/conf.py b/docs/source/conf.py index 5caeb70b6..3945e9eed 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -41,11 +41,19 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', - 'sphinx.ext.napoleon', 'recommonmark', 'sphinx_markdown_tables', - 'post_process' + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.mathjax', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', + 'sphinx.ext.napoleon', + 'recommonmark', + 'sphinx_markdown_tables', + 'post_process', ] # Add any paths that contain templates here, relative to this directory. @@ -66,7 +74,7 @@ # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. # language = u'zh_CN' -language = u'en' +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -118,15 +126,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -136,7 +141,7 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'easy_rec.tex', u'easy\\_rec Documentation', u'EasyRec Team', + (master_doc, 'easy_rec.tex', 'easy\\_rec Documentation', 'EasyRec Team', 'manual'), ] @@ -144,7 +149,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [(master_doc, 'easy_rec', u'easy_rec Documentation', [author], 1)] +man_pages = [(master_doc, 'easy_rec', 'easy_rec Documentation', [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -152,8 +157,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'easy_rec', u'easy_rec Documentation', author, 'easy_rec', - 'One line description of project.', 'Miscellaneous'), + ( + master_doc, + 'easy_rec', + 'easy_rec Documentation', + author, + 'easy_rec', + 'One line description of project.', + 'Miscellaneous', + ), ] # -- Options for Epub output ------------------------------------------------- diff --git a/docs/source/export.md b/docs/source/export.md index 9ab9b31d4..fd09fbb5f 100644 --- a/docs/source/export.md +++ b/docs/source/export.md @@ -8,7 +8,7 @@ export_config { ``` - batch_size: 导出模型的batch_size，默认是-1，即可以接收任意batch_size -- exporter_type: 导出类型, best | final | latest | none，默认final +- exporter_type: 导出类型, best | final | latest | none，默认final - best 导出最好的模型 - final 训练结束后导出 - latest 导出最新的模型 @@ -88,8 +88,8 @@ pai -name easy_rec_ext -project algo_public - -Dconfig: 同训练 - -Dcmd: export 模型导出 - -Dexport_dir: 导出的目录 -- -Dcheckpoint_path: 使用指定的checkpoint_path -- -Darn: rolearn  注意这个的arn要替换成客户自己的。可以从dataworks的设置中查看arn。 +- -Dcheckpoint_path: 可选参数，使用指定的checkpoint_path导出 +- -Darn: rolearn 注意这个的arn要替换成客户自己的。可以从dataworks的设置中查看arn。 - -DossHost: ossHost地址 - -Dbuckets: config所在的bucket和保存模型的bucket; 如果有多个bucket，逗号分割 - 如果是pai内部版,则不需要指定arn和ossHost, arn和ossHost放在-Dbuckets里面 @@ -98,6 +98,9 @@ pai -name easy_rec_ext -project algo_public - --export_done_file: 导出完成标志文件名, 导出完成后，在导出目录下创建一个文件表示导出完成了 - --clear_export: 删除旧的导出文件目录 - --place_embedding_on_cpu: 将embedding相关的操作放在cpu上，有助于提升模型在gpu环境下的推理速度 + - --asset_files: 需要导出的asset文件路径, 可设置多个, 逗号分隔； + - 如果需要导出到assets目录的子目录下，使用`${target_path}:${source_path}`的格式；（从版本0.8.7开始支持） + - e.g. '--asset_files custom_fg_lib/fg.json:oss://${bucket}/path/to/fg.json' - 模型导出之后可以使用(EasyRecProcessor)\[./predict/在线预测.md\]部署到PAI-EAS平台 ### 双塔召回模型 diff --git a/docs/source/feature/data.md b/docs/source/feature/data.md index 169902b78..7e424dc3e 100644 --- a/docs/source/feature/data.md +++ b/docs/source/feature/data.md @@ -79,8 +79,10 @@ process_lbl.py: ```python import numpy as np + + def remap_lbl(labels): - res = np.where(labels<5, 0, 1) + res = np.where(labels < 5, 0, 1) return res ``` diff --git a/docs/source/feature/excel_config.md b/docs/source/feature/excel_config.md index 49c4def69..19900f1ba 100644 --- a/docs/source/feature/excel_config.md +++ b/docs/source/feature/excel_config.md @@ -27,12 +27,12 @@ python -m easy_rec.python.tools.create_config_from_excel --model_type multi_towe - 输入输出文件 - --train_input_path TRAIN_INPUT_PATH --eval_input_path EVAL_INPUT_PATH + --train_input_path TRAIN_INPUT_PATH --eval_input_path EVAL_INPUT_PATH - 默认数据文件(csv)列(column)分割符号是, 列(column)内部里面字符分割符号是| 可以自定义分隔符: - --column_separator $'|' --incol_separator $',' + --column_separator $'|' --incol_separator $',' - 训练数据路径 --train_input_path diff --git a/docs/source/feature/fg.md b/docs/source/feature/fg.md index cdc7a1b13..c69a1cfc5 100644 --- a/docs/source/feature/fg.md +++ b/docs/source/feature/fg.md @@ -23,7 +23,7 @@ FG模块在推荐系统架构中的位置如下图所示： #### 编写配置文件`fg.json` -- 包含了features配置和全局配置两个部分, 示例: +- 包含了features配置和全局配置两个部分, 示例: ```json { @@ -182,7 +182,7 @@ eascmd -i -k -e update ali_rec_rn - processor: easyrec processor, 目前最新的版本为easyrec-3.0, [历史版本](../predict/processor.md#release). - model_config: eas 部署配置。主要控制把 item 特征加载到内存中。目前数据源支持redis和holo - period: item feature reload period, 单位minutes - - url: holo url, 格式为postgresql://:@<域名>:/ + - url: holo url, 格式为postgresql://:@\<域名>:/ - fg_mode: 支持tf和normal两种模式, tf模式表示fg是以TF算子的方式执行的, 性能更好 - tables: item特征存储在hologres表里面, 支持分多个表存储 - key: 必填, itemId列的名字; @@ -191,7 +191,7 @@ eascmd -i -k -e update ali_rec_rn - timekey: 可选，用于item的增量更新，支持的格式: timestamp和int - static: 可选, 表示是静态特征，不用周期性更新 - 支持多个item表, 如果多张表有重复的列, 后面的表覆盖前面的表 - - "tables": [{"key":"table1", ...},{"key":"table2", ...}] + - "tables": \[{"key":"table1", ...},{"key":"table2", ...}\] - 如果多张表有重复的列，后面的表将覆盖前面的表 - hologres表里面每一列存储一个item特征,示例: @@ -202,7 +202,7 @@ eascmd -i -k -e update ali_rec_rn
- remote_type: Item特征数据源, 目前支持：`hologres`, `none` - hologres：通过SQL接口进行数据读取和写入，适用于海量数据的存储和查询 - - none: 不使用Item特征缓存，item特征通过请求传入，此时tables应设置为[] + - none: 不使用Item特征缓存，item特征通过请求传入，此时tables应设置为\[\] - storage: 将oss的模型目录mount到docker的指定目录下 - mount_path: docker内部的挂载路径, 与示例保持一致即可 - 配置了storage就不需要配置model_path了 diff --git a/docs/source/feature/fg_docs/RawFeature.md b/docs/source/feature/fg_docs/RawFeature.md index 45f732f78..9159d315d 100644 --- a/docs/source/feature/fg_docs/RawFeature.md +++ b/docs/source/feature/fg_docs/RawFeature.md @@ -26,11 +26,11 @@ raw_feature表示连续值特征, 支持数值int、float、double等数值类 ^\]表示多值分隔符，注意这是一个符号，其ASCII编码是"\\x1D"，而不是两个符号 -| 类型 | item:ctr的取值 | 输出的feature | -| ------- | ----------- | ---------------------------------------------- | -| int64_t | 100 | (ctr, 100) | -| double | 100.1 | (ctr, 100.1) | -| 多值int | 123^\]456 | (ctr, (123,456)) (注意，输入字段必须与配置的dimension维度一致) | +| 类型 | item:ctr的取值 | 输出的feature | +| ------- | ----------- | --------------------------------------------- | +| int64_t | 100 | (ctr, 100) | +| double | 100.1 | (ctr, 100.1) | +| 多值int | 123^\]456 | (ctr, (123,456)) (注意，输入字段必须与配置的dimension维度一致) | ## Normalizer diff --git a/docs/source/feature/odl_sample.md b/docs/source/feature/odl_sample.md index 0fe6cdda8..493ad5081 100644 --- a/docs/source/feature/odl_sample.md +++ b/docs/source/feature/odl_sample.md @@ -202,7 +202,7 @@ 1. label生成, 目前提供三种[python udf](http://easyrec.oss-cn-beijing.aliyuncs.com/deploy/label_gen.zip): - playtime: sum_over(events, 'playtime') - - click: has_event(events, 'click') + - click: has_event(events, 'click') - min_over / max_over: min_over(events, 'eventTime') - 可以使用python自定义任意udf, [参考文档](https://flink.apache.org/2020/04/09/pyflink-udf-support-flink.html) - udf 上传([vvp-console](https://vvp.console.aliyun.com/)): diff --git a/docs/source/feature/rtp_fg.md b/docs/source/feature/rtp_fg.md index 40da4852e..ef3f66e11 100644 --- a/docs/source/feature/rtp_fg.md +++ b/docs/source/feature/rtp_fg.md @@ -12,7 +12,7 @@ #### 编写配置 [fg.json](https://easyrec.oss-cn-beijing.aliyuncs.com/rtp_fg/fg.json) -- 包含了features配置和全局配置两个部分, 示例: +- 包含了features配置和全局配置两个部分, 示例: ```json { @@ -325,7 +325,7 @@ python -m easy_rec.python.tools.convert_rtp_fg --label is_product_detail is_pur - --separator: feature之间的分隔符, 默认是CTRL_B(u0002) -- --selected_cols: 指定输入列，包括label、\[sample_weight\]和features，其中label可以指定多列，表示要使用多个label(一般是多任务模型), 最后一列必须是features, 如: +- --selected_cols: 指定输入列，包括label、\[sample_weight\]和features，其中label可以指定多列，表示要使用多个label(一般是多任务模型), 最后一列必须是features, 如: ``` label0,label1,sample_weight,features diff --git a/docs/source/feature/rtp_native.md b/docs/source/feature/rtp_native.md index 8774041c7..7d3730674 100644 --- a/docs/source/feature/rtp_native.md +++ b/docs/source/feature/rtp_native.md @@ -6,7 +6,7 @@ #### 编写RTP特征配置 [fg.json](https://easyrec.oss-cn-beijing.aliyuncs.com/rtp_fg/fg.json) -- 包含了features配置和全局配置两个部分, 示例: +- 包含了features配置和全局配置两个部分, 示例: ```json { diff --git a/docs/source/incremental_train.md b/docs/source/incremental_train.md index c2c82f782..6f72a9bf7 100644 --- a/docs/source/incremental_train.md +++ b/docs/source/incremental_train.md @@ -57,4 +57,4 @@ pai -name easy_rec_ext -project algo_public ``` - bizdate在dataworks里面是业务日期，一般是运行日期的前一天。 -- fine_tune_checkpoint: fine_tune_checkpoint的路径，可以指定具体的checkpoint，也可以指定一个目录，将自动定位目录里面最新的checkpoint。 +- fine_tune_checkpoint: fine_tune_checkpoint的路径，可以指定具体的checkpoint，也可以指定一个目录，将自动定位目录里面最新的checkpoint。 diff --git a/docs/source/intro.md b/docs/source/intro.md index 65844ac8d..1f1cc70a0 100644 --- a/docs/source/intro.md +++ b/docs/source/intro.md @@ -1,6 +1,6 @@ # EasyRec简介 -🎉 See our ongoing recommendation framework **[TorchEasyRec](https://github.com/alibaba/TorchEasyRec) !** 🎉 This evolution of EasyRec is built on **PyTorch**, featuring **GPU acceleration** and **hybrid parallelism** for enhanced performance. +🎉 See our ongoing recommendation framework **[TorchEasyRec](https://github.com/alibaba/TorchEasyRec) !** 🎉 This evolution of EasyRec is built on **PyTorch**, featuring **GPU acceleration** and **hybrid parallelism** for enhanced performance. ## What is EasyRec? diff --git a/docs/source/models/aitm.md b/docs/source/models/aitm.md index 6f4c57d7b..036567e50 100644 --- a/docs/source/models/aitm.md +++ b/docs/source/models/aitm.md @@ -8,7 +8,7 @@ 1. (a) Expert-Bottom pattern。如 [MMoE](mmoe.md) 1. (b) Probability-Transfer pattern。如 [ESMM](esmm.md) -1. (c) Adaptive Information Transfer Multi-task (AITM) framework. +1. (c) Adaptive Information Transfer Multi-task (AITM) framework. 两个特点： diff --git a/docs/source/models/deepfm.md b/docs/source/models/deepfm.md index 7dfe58418..c1c8ecc5f 100644 --- a/docs/source/models/deepfm.md +++ b/docs/source/models/deepfm.md @@ -49,7 +49,7 @@ model_config:{ - model_class: 'DeepFM', 不需要修改 - feature_groups: 需要两个feature_group: wide group和deep group, **group name不能变** -- deepfm: deepfm相关的参数 +- deepfm: deepfm相关的参数 - dnn: deep part的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - wide_output_dim: wide部分输出的大小 diff --git a/docs/source/models/esmm.md b/docs/source/models/esmm.md index c12c78dae..959232ae1 100644 --- a/docs/source/models/esmm.md +++ b/docs/source/models/esmm.md @@ -94,7 +94,7 @@ model_config: { - feature_groups: 支持多组feature_group - esmm: esmm相关的参数 - groups - - input tower的input必须和feature_groups的group_name对应 + - input tower的input必须和feature_groups的group_name对应 - dnn deep part的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - cvr_tower @@ -114,7 +114,7 @@ model_config: { ESMM模型输出的值有以下几项： - "logits\_" / "probs\_" + ctr_tower的tower_name -- "logits\_" / "probs\_" / "y\_" + cvr_tower的tower_name +- "logits\_" / "probs\_" / "y\_" + cvr_tower的tower_name - "probs_ctcvr" / "y_ctcvr" ESMM模型的指标有以下几项： diff --git a/docs/source/models/loss.md b/docs/source/models/loss.md index e098aa0a6..f5c9cee32 100644 --- a/docs/source/models/loss.md +++ b/docs/source/models/loss.md @@ -53,7 +53,7 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 下面的配置可以同时使用`F1_REWEIGHTED_LOSS`和`PAIR_WISE_LOSS`，总的loss为这两个损失函数的加权求和。 -``` +```protobuf losses { loss_type: F1_REWEIGHTED_LOSS weight: 1.0 @@ -71,7 +71,7 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 可以调节二分类模型recall/precision相对权重的损失函数，配置如下： - ``` + ```protobuf { loss_type: F1_REWEIGHTED_LOSS f1_reweight_loss { @@ -134,6 +134,10 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 - session_name: list分组的字段名，比如user_id - 参考论文：《 [Joint Optimization of Ranking and Calibration with Contextualized Hybrid Model](https://arxiv.org/pdf/2208.06164.pdf) 》 - 使用示例: [dbmtl_with_jrc_loss.config](https://github.com/alibaba/EasyRec/blob/master/samples/model_config/dbmtl_on_taobao_with_multi_loss.config) + - 有几个注意点： + 1. JRC_Loss不要和普通二分类loss一起使用，因为它内部已经包含了二分类loss了，最好是先单独使用 + 1. JRC_Loss依赖mini-batch类的同session样本对，因此样本不能全局随机打散； 要把样本按照session_id 分组，同一组的样本需要shuffle到一起；（考验sql功力，如果搞不定不分组也可以，但需要保证同一个session的样本尽量排在一起，即group by session_id） + 1. 模型训练时`batch_size`尽可能大，在内存能够支撑的前提下`batch_size`调到最大（比如，8192）之后，再调整其他参数（如果需要的话） - LISTWISE_RANK_LOSS 的参数配置 @@ -142,6 +146,33 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 - label_is_logits: bool, 标记label是否为teacher模型的输出logits，默认为false - scale_logits: bool, 是否需要对模型的logits进行线性缩放，默认为false +- ZILN_LOSS 的参数配置 + + - mu_regularization: mu参数的正则化系数，默认值为0.01 + - sigma_regularization: sigma参数的正则化系数，默认值为0.01 + - max_sigma: sigma参数的最大值，默认值为5.0（sigma>5 就会让均值乘上 exp(0.5\*25) ≈ 2.7e5 的因子，已经很激进） + - max_log_clip_value: log(预测值)的最大值，默认值为20.0（最大预测值默认为exp(20)） + - return_log_pred_value: 是否返回log(预测值)，默认值为false + - classification_weight: 分类任务的权重，默认值为1.0 + - regression_weight: 回归任务的权重，默认值为1.0；零值越多，建议分类权重越小，回归权重越大 + - 配置示例如下 + ```protobuf + losses { + loss_type: ZILN_LOSS + weight: 1.0 + loss_name: "LTV" + ziln_loss { + mu_regularization: 0.01 + sigma_regularization: 0.01 + max_log_clip_value: 20.0 + max_sigma: 5.0 + return_log_pred_value: false + classification_weight: 1.0 + regression_weight: 1.0 + } + } + ``` + 排序模型同时使用多个损失函数的完整示例： [cmbf_with_multi_loss.config](https://github.com/alibaba/EasyRec/blob/master/samples/model_config/cmbf_with_multi_loss.config) @@ -183,6 +214,31 @@ EasyRec支持两种损失函数配置方式：1）使用单个损失函数；2 - loss_weight_strategy: Random - 表示损失函数的权重设定为归一化的随机数 +### 根据样本设定损失函数权重（Masked Loss） + +多目标学习任务中，通常需要根据样本的属性来设定损失函数的权重。 + +#### 根据样本属性设定损失函数权重 + +在某个目标的tower里配置`task_space_indicator_name`和`task_space_indicator_value` + +- task_space_indicator_name 是特征名 +- task_space_indicator_value 是特征值 +- in_task_space_weight 目标样本的loss权重，默认值为1.0 +- out_task_space_weight 非目标样本的loss权重，默认值为1.0 + +如果样本的特征值与你配置的`task_space_indicator_value`相等, loss 权重 * `in_task_space_weight`; +不相等则loss权重 * `out_task_space_weight`。 + +`out_task_space_weight`的值改为0.0，则可实现`Masked Loss`。 + +#### 根据样本label来设定损失函数权重 + +在某个目标的tower里配置`task_space_indicator_label`这个字段，标记一个 label 的名字， +如果这个label的值大于0, 则 loss 权重 \*`in_task_space_weight`; 否则 loss权重 * `out_task_space_weight`。 + +`out_task_space_weight`的值改为0.0，则可实现`Masked Loss`。 + ### 参考论文： - 《 Multi-Task Learning Using Uncertainty to Weigh Losses for Scene Geometry and Semantics 》 diff --git a/docs/source/models/mind.md b/docs/source/models/mind.md index 8a52d6a83..3553ae3fd 100644 --- a/docs/source/models/mind.md +++ b/docs/source/models/mind.md @@ -129,7 +129,7 @@ model_config:{ - time_id取值的方式可参考: - - 训练数据: Math.round((2 * Math.log1p((labelTime - itemTime) / 60.) / Math.log(2.))) + 1 + - 训练数据: Math.round((2 * Math.log1p((labelTime - itemTime) / 60.) / Math.log(2.))) + 1 - inference: Math.round((2 * Math.log1p((currentTime - itemTime) / 60.) / Math.log(2.))) + 1 - 此处的时间(labelTime, itemTime, currentTime) 为seconds diff --git a/docs/source/models/rocket_launching.md b/docs/source/models/rocket_launching.md index 172319e6d..b97af8ed1 100644 --- a/docs/source/models/rocket_launching.md +++ b/docs/source/models/rocket_launching.md @@ -59,7 +59,7 @@ model_config: { - hidden_units: dnn每一层的channel数目，即神经元的数目 - booster_dnn: 助推器网络的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - - light_dnn: 轻量网络的参数配置 + - light_dnn: 轻量网络的参数配置 - hidden_units: dnn每一层的channel数目，即神经元的数目 - feature_based_distillation：是否配置特征蒸馏（默认不配） - feature_distillation_function：中间层相似度衡量指标（COSINE; EUCLID; 默认COSINE） diff --git a/docs/source/models/user_define.md b/docs/source/models/user_define.md index 4272fcf60..4e48214d0 100644 --- a/docs/source/models/user_define.md +++ b/docs/source/models/user_define.md @@ -298,12 +298,11 @@ python -m easy_rec.python.train_eval --pipeline_config_path samples/model_config 增加测试用例到easy_rec/python/test/train_eval_test.py ```python - def test_custom_model(self): +def test_custom_model(self): self._success = test_utils.test_single_train_eval( - 'samples/model_config/custom_model.config', - self._test_dir) + "samples/model_config/custom_model.config", self._test_dir + ) self.assertTrue(self._success) - ``` 运行CustomModel测试用例 diff --git a/docs/source/models/wide_and_deep.md b/docs/source/models/wide_and_deep.md index a225b58d6..d09dc6ed9 100644 --- a/docs/source/models/wide_and_deep.md +++ b/docs/source/models/wide_and_deep.md @@ -52,7 +52,7 @@ model_config:{ 需要两个feature_group: wide group和deep group, **group name不能变** -- wide_and_deep: wide_and_deep 相关的参数 +- wide_and_deep: wide_and_deep 相关的参数 - dnn: deep part的参数配置 diff --git a/docs/source/online_train.md b/docs/source/online_train.md index 3af5d05cf..981a31cef 100644 --- a/docs/source/online_train.md +++ b/docs/source/online_train.md @@ -84,7 +84,7 @@ datahub_eval_input{ - akSecret: datahub access_secret -- region: endpoint +- region: endpoint - 注意必须是http的，不能是https，vpc里面只通80端口，不通443端口 @@ -140,7 +140,7 @@ train_config { } ``` -- dense_save_steps: dense参数发送的频率 +- dense_save_steps: dense参数发送的频率 - sparse_save_steps: sparse参数发送的频率 - incr_update: 保存增量更新，可以选fs和kafka: - fs: diff --git a/docs/source/predict/processor.md b/docs/source/predict/processor.md index 213063940..754d55130 100644 --- a/docs/source/predict/processor.md +++ b/docs/source/predict/processor.md @@ -24,8 +24,8 @@ FeatureGenerator作为算子嵌入, 和TFModel联合优化，主要的优化点 - 在线推理User Feature的BatchSize = 1, 因此只需要计算一次, Tile可以放到UserFeature EmbeddingLookUp的后面，和EmbeddingLookUp融合起来 - EmbeddingLookup OpFusion - embedding lookup涉及到GatherV2 / SparseSegmentMean等多个小op, 使用OpFusion可以减少kernel launch以及Tensor内存分配和拷贝的开销 - - embedding lookup的过程需要用到比较多的SparseSegment Mean/Sum操作, 使用avx指令优化向量运算 -- MatchFeature / LookupFeature优化 + - embedding lookup的过程需要用到比较多的SparseSegment Mean/Sum操作, 使用avx指令优化向量运算 +- MatchFeature / LookupFeature优化 - 使用string_view代替std::string，减少string拷贝的开销 - Sequence特征优化 - sequence特征建模通常需要带上side info才能取得比较好的效果, 将side info放在请求中传递过来会带来通信的开销, EasyRec Processor在客户侧缓存了item特征, 因此在请求中只传递item_id sequence, side info sequence通过item_id查找客户侧缓存构建. @@ -39,7 +39,7 @@ FeatureGenerator作为算子嵌入, 和TFModel联合优化，主要的优化点 - 特征: - +
id_featureraw_featurelookup_featuresequence_feature
67 170 756 main seq num: 4 | sideinfo seq num: 32 | max seq len: 50
@@ -52,7 +52,7 @@ FeatureGenerator作为算子嵌入, 和TFModel联合优化，主要的优化点 - CPU型号: [IceLake](https://help.aliyun.com/document_detail/68564.html#p-zpg-gvj-g91) - 测试结果: - +
@@ -63,7 +63,7 @@ FeatureGenerator作为算子嵌入, 和TFModel联合优化，主要的优化点 - 特征数: -
CPU利用率QPSAVG RTTP99
优化前96 20 247ms 333ms
优化后91 55 86ms 113ms
+
id_feature raw_feature lookup_feature match_feature
306 77 60 1000
@@ -76,7 +76,7 @@ FeatureGenerator作为算子嵌入, 和TFModel联合优化，主要的优化点 - CPU型号: [IceLake](https://help.aliyun.com/document_detail/68564.html#p-zpg-gvj-g91) - 测试结果: - +
diff --git a/docs/source/quick_start/emr_tutorial.md b/docs/source/quick_start/emr_tutorial.md index 7d19e50fb..7e8f45db9 100644 --- a/docs/source/quick_start/emr_tutorial.md +++ b/docs/source/quick_start/emr_tutorial.md @@ -158,7 +158,7 @@ resource: - 使用el_submit提交导出任务, **el_submit**相关参数请参考[**tf_on_yarn**](https://help.aliyun.com/document_detail/93031.html) --pipeline_config_path: EasyRec配置文件 ---export_dir: 导出模型目录  +--export_dir: 导出模型目录 --checkpoint_path: 指定checkpoint，默认不指定，不指定则使用model_dir下面最新的checkpoint ### 开源TF模式 diff --git a/docs/source/quick_start/mc_tutorial.md b/docs/source/quick_start/mc_tutorial.md index 0f6065f0c..14e6fedef 100644 --- a/docs/source/quick_start/mc_tutorial.md +++ b/docs/source/quick_start/mc_tutorial.md @@ -11,7 +11,7 @@ 输入一般是MaxCompute表: - train: pai_online_project.dwd_avazu_ctr_deepmodel_train -- test: pai_online_project.dwd_avazu_ctr_deepmodel_test +- test: pai_online_project.dwd_avazu_ctr_deepmodel_test 说明：原则上这两张表是自己odps的表，为了方便，以上提供case的两张表可在国内用户的MaxCompute项目空间中访问。 @@ -49,13 +49,13 @@ pai -name easy_rec_ext -project algo_public - -Dfine_tune_checkpoint: 可选，从checkpoint restore参数，进行finetune - 可以指定directory，将使用directory里面的最新的checkpoint. - -Dmodel_dir: 如果指定了model_dir将会覆盖config里面的model_dir，一般在周期性调度的时候使用。 -- -Darn: rolearn 注意这个的arn要替换成客户自己的。可以从dataworks的设置中查看arn;或者阿里云控制台人工智能平台PAI，左侧菜单"开通和授权"，找到全部云产品依赖->Designer->OSS->查看授权信息。 +- -Darn: rolearn 注意这个的arn要替换成客户自己的。可以从dataworks的设置中查看arn;或者阿里云控制台人工智能平台PAI，左侧菜单"开通和授权"，找到全部云产品依赖->Designer->OSS->查看授权信息。 - -Dbuckets: config所在的bucket和保存模型的bucket; 如果有多个bucket，逗号分割 - -DossHost: ossHost地址 ### 注意： -- dataworks和PAI的project一样，案例都是pai_online_project，用户需要根据自己的环境修改。如果需要使用gpu，PAI的project需要设置开通GPU。链接：[https://pai.data.aliyun.com/console?projectId=®ionId=cn-beijing#/visual](https://pai.data.aliyun.com/console?projectId=%C2%AEionId=cn-beijing#/visual) ，其中regionId可能不一致。 +- dataworks和PAI的project一样，案例都是pai_online_project，用户需要根据自己的环境修改。如果需要使用gpu，PAI的project需要设置开通GPU。链接：[https://pai.data.aliyun.com/console?projectId=®ionId=cn-beijing#/visual](https://pai.data.aliyun.com/console?projectId=%C2%AEionId=cn-beijing#/visual) ，其中regionId可能不一致。 ![mc_gpu](../../images/quick_start/mc_gpu.png) diff --git a/docs/source/quick_start/mc_tutorial_inner.md b/docs/source/quick_start/mc_tutorial_inner.md index d04dc2e8d..5870c6c1b 100644 --- a/docs/source/quick_start/mc_tutorial_inner.md +++ b/docs/source/quick_start/mc_tutorial_inner.md @@ -57,7 +57,7 @@ pai -name easy_rec_ext -project algo_public ### 注意： -- dataworks和pai的project 一样，案例都是pai_online_project，用户需要根据自己的环境修改。如果需要使用gpu，PAI的project需要设置开通GPU。链接：[https://pai.data.aliyun.com/console?projectId=®ionId=cn-beijing#/visual](https://pai.data.aliyun.com/console?projectId=%C2%AEionId=cn-beijing#/visual) ，其中regionId可能不一致。 +- dataworks和pai的project 一样，案例都是pai_online_project，用户需要根据自己的环境修改。如果需要使用gpu，PAI的project需要设置开通GPU。链接：[https://pai.data.aliyun.com/console?projectId=®ionId=cn-beijing#/visual](https://pai.data.aliyun.com/console?projectId=%C2%AEionId=cn-beijing#/visual) ，其中regionId可能不一致。 ![mc_gpu](../../images/quick_start/mc_gpu.png) diff --git a/docs/source/train.md b/docs/source/train.md index 15907d3ab..1130c5ea0 100644 --- a/docs/source/train.md +++ b/docs/source/train.md @@ -2,9 +2,9 @@ ## train_config -- log_step_count_steps: 200 # 每200步打印一行log +- log_step_count_steps: 200 # 每200步打印一行log -- optimizer_config # 优化器相关的参数 +- optimizer_config # 优化器相关的参数 ```protobuf { @@ -55,12 +55,12 @@ - 设置两个optimizer时, 第一个optimizer仅用于wide参数; - 如果要给deep embedding单独设置optimizer, 需要设置3个optimizer. -- sync_replicas: true # 是否同步训练，默认是false +- sync_replicas: true # 是否同步训练，默认是false - 使用SyncReplicasOptimizer进行分布式训练(同步模式) - 仅在train_distribute为NoStrategy时可以设置成true，其它情况应该设置为false - PS异步训练也设置为false - - 注意在设置为 true 时，总共的训练步数为：min(total_sample_num \* num_epochs / batch_size, num_steps) / num_workers + - 注意在设置为 true 时，总共的训练步数为：min(total_sample_num * num_epochs / batch_size, num_steps) / num_workers - train_distribute: 默认不开启Strategy(NoStrategy), strategy确定分布式执行的方式, 可以分成两种模式: PS-Worker模式 和 All-Reduce模式 @@ -101,10 +101,10 @@ import tensorflow as tf import os, sys - ckpt_reader = tf.train.NewCheckpointReader('experiments/model.ckpt-0') + ckpt_reader = tf.train.NewCheckpointReader("experiments/model.ckpt-0") ckpt_var2shape_map = ckpt_reader.get_variable_to_shape_map() for key in ckpt_var2shape_map: - print(key) + print(key) ``` - save_checkpoints_steps: 每隔多少步保存一次checkpoint, 默认是1000。当训练数据量很大的时候，这个值要设置大一些 diff --git a/easy_rec/__init__.py b/easy_rec/__init__.py index cbbc20e9c..a81a50c0d 100644 --- a/easy_rec/__init__.py +++ b/easy_rec/__init__.py @@ -18,11 +18,13 @@ # Avoid import tensorflow which conflicts with the version used in EasyRecProcessor if 'PROCESSOR_TEST' not in os.environ: from tensorflow.python.platform import tf_logging + # In DeepRec, logger.propagate of tf_logging is False, should be True tf_logging._logger.propagate = True def get_ops_dir(): import tensorflow as tf + if platform.system() == 'Linux': ops_dir = os.path.join(curr_dir, 'python/ops') if 'PAI' in tf.__version__: @@ -47,7 +49,8 @@ def get_ops_dir(): logging.warning('ops_dir[%s] does not exist' % ops_dir) ops_dir = None - from easy_rec.python.inference.predictor import Predictor # isort:skip # noqa: E402 + from easy_rec.python.inference.predictor import ( # isort:skip # noqa: E402 + Predictor,) from easy_rec.python.main import evaluate # isort:skip # noqa: E402 from easy_rec.python.main import distribute_evaluate # isort:skip # noqa: E402 from easy_rec.python.main import export # isort:skip # noqa: E402 diff --git a/easy_rec/python/builders/hyperparams_builder.py b/easy_rec/python/builders/hyperparams_builder.py index a1caf537e..e018d94c4 100644 --- a/easy_rec/python/builders/hyperparams_builder.py +++ b/easy_rec/python/builders/hyperparams_builder.py @@ -14,6 +14,7 @@ # limitations under the License. # ============================================================================== """Builder function to construct tf-slim arg_scope for convolution, fc ops.""" + import tensorflow as tf from easy_rec.python.compat import regularizers @@ -44,7 +45,8 @@ def build_regularizer(regularizer): if regularizer_oneof == 'l1_l2_regularizer': return regularizers.l1_l2_regularizer( scale_l1=float(regularizer.l1_l2_regularizer.scale_l1), - scale_l2=float(regularizer.l1_l2_regularizer.scale_l2)) + scale_l2=float(regularizer.l1_l2_regularizer.scale_l2), + ) raise ValueError('Unknown regularizer function: {}'.format(regularizer_oneof)) @@ -65,11 +67,13 @@ def build_initializer(initializer): if initializer_oneof == 'truncated_normal_initializer': return tf.truncated_normal_initializer( mean=initializer.truncated_normal_initializer.mean, - stddev=initializer.truncated_normal_initializer.stddev) + stddev=initializer.truncated_normal_initializer.stddev, + ) if initializer_oneof == 'random_normal_initializer': return tf.random_normal_initializer( mean=initializer.random_normal_initializer.mean, - stddev=initializer.random_normal_initializer.stddev) + stddev=initializer.random_normal_initializer.stddev, + ) if initializer_oneof == 'glorot_normal_initializer': return tf.glorot_normal_initializer() if initializer_oneof == 'constant_initializer': diff --git a/easy_rec/python/builders/loss_builder.py b/easy_rec/python/builders/loss_builder.py index 720dfdd9e..1914152ca 100644 --- a/easy_rec/python/builders/loss_builder.py +++ b/easy_rec/python/builders/loss_builder.py @@ -7,17 +7,19 @@ from easy_rec.python.loss.focal_loss import sigmoid_focal_loss_with_logits from easy_rec.python.loss.jrc_loss import jrc_loss -from easy_rec.python.loss.listwise_loss import listwise_distill_loss -from easy_rec.python.loss.listwise_loss import listwise_rank_loss -from easy_rec.python.loss.pairwise_loss import pairwise_focal_loss -from easy_rec.python.loss.pairwise_loss import pairwise_hinge_loss -from easy_rec.python.loss.pairwise_loss import pairwise_logistic_loss -from easy_rec.python.loss.pairwise_loss import pairwise_loss from easy_rec.python.protos.loss_pb2 import LossType -from easy_rec.python.loss.zero_inflated_lognormal import zero_inflated_lognormal_loss # NOQA - -from easy_rec.python.loss.f1_reweight_loss import f1_reweight_sigmoid_cross_entropy # NOQA +from easy_rec.python.loss.f1_reweight_loss import ( # NOQA + f1_reweight_sigmoid_cross_entropy,) +from easy_rec.python.loss.listwise_loss import ( # NOQA + listwise_distill_loss, listwise_rank_loss, +) +from easy_rec.python.loss.pairwise_loss import ( # NOQA + pairwise_focal_loss, pairwise_hinge_loss, pairwise_logistic_loss, + pairwise_loss, +) +from easy_rec.python.loss.zero_inflated_lognormal import ( # NOQA + zero_inflated_lognormal_loss,) if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -36,8 +38,10 @@ def build(loss_type, return tf.losses.sigmoid_cross_entropy( label, logits=pred, weights=loss_weight, **kwargs) else: - assert label.dtype in [tf.int32, tf.int64], \ - 'label.dtype must in [tf.int32, tf.int64] when use sparse_softmax_cross_entropy.' + assert label.dtype in [ + tf.int32, + tf.int64, + ], 'label.dtype must in [tf.int32, tf.int64] when use sparse_softmax_cross_entropy.' return tf.losses.sparse_softmax_cross_entropy( labels=label, logits=pred, weights=loss_weight, **kwargs) elif loss_type == LossType.CROSS_ENTROPY_LOSS: @@ -50,7 +54,23 @@ def build(loss_type, return tf.losses.mean_squared_error( labels=label, predictions=pred, weights=loss_weight, **kwargs) elif loss_type == LossType.ZILN_LOSS: - loss = zero_inflated_lognormal_loss(label, pred) + if loss_param is None: + loss = zero_inflated_lognormal_loss(label, pred) + else: + mu_reg = loss_param.mu_regularization + sigma_reg = loss_param.sigma_regularization + max_sigma = loss_param.max_sigma + class_weight = loss_param.classification_weight + reg_weight = loss_param.regression_weight + loss = zero_inflated_lognormal_loss( + label, + pred, + max_sigma=max_sigma, + mu_reg=mu_reg, + sigma_reg=sigma_reg, + class_weight=class_weight, + reg_weight=reg_weight, + ) if np.isscalar(loss_weight) and loss_weight != 1.0: return loss * loss_weight return loss @@ -66,7 +86,8 @@ def build(loss_type, loss_weight_strategy=loss_param.loss_weight_strategy, sample_weights=loss_weight, same_label_loss=loss_param.same_label_loss, - name=loss_name) + name=loss_name, + ) elif loss_type == LossType.PAIR_WISE_LOSS: session = kwargs.get('session_ids', None) margin = 0 if loss_param is None else loss_param.margin @@ -78,7 +99,8 @@ def build(loss_type, margin=margin, temperature=temp, weights=loss_weight, - name=loss_name) + name=loss_name, + ) elif loss_type == LossType.PAIRWISE_LOGISTIC_LOSS: session = kwargs.get('session_ids', None) temp = 1.0 if loss_param is None else loss_param.temperature @@ -96,7 +118,8 @@ def build(loss_type, ohem_ratio=ohem_ratio, weights=loss_weight, use_label_margin=lbl_margin, - name=loss_name) + name=loss_name, + ) elif loss_type == LossType.PAIRWISE_HINGE_LOSS: session = kwargs.get('session_ids', None) temp, ohem_ratio, margin = 1.0, 1.0, 1.0 @@ -119,7 +142,8 @@ def build(loss_type, label_is_logits=label_is_logits, use_label_margin=use_label_margin, use_exponent=use_exponent, - name=loss_name) + name=loss_name, + ) elif loss_type == LossType.PAIRWISE_FOCAL_LOSS: session = kwargs.get('session_ids', None) if loss_param is None: @@ -138,7 +162,8 @@ def build(loss_type, ohem_ratio=loss_param.ohem_ratio, temperature=loss_param.temperature, weights=loss_weight, - name=loss_name) + name=loss_name, + ) elif loss_type == LossType.LISTWISE_RANK_LOSS: session = kwargs.get('session_ids', None) trans_fn, temp, label_is_logits, scale = None, 1.0, False, False @@ -156,7 +181,8 @@ def build(loss_type, label_is_logits=label_is_logits, transform_fn=trans_fn, scale_logits=scale, - weights=loss_weight) + weights=loss_weight, + ) elif loss_type == LossType.LISTWISE_DISTILL_LOSS: session = kwargs.get('session_ids', None) trans_fn, temp, label_clip_max_value, scale = None, 1.0, 512.0, False @@ -174,7 +200,8 @@ def build(loss_type, label_clip_max_value=label_clip_max_value, transform_fn=trans_fn, scale_logits=scale, - weights=loss_weight) + weights=loss_weight, + ) elif loss_type == LossType.F1_REWEIGHTED_LOSS: f1_beta_square = 1.0 if loss_param is None else loss_param.f1_beta_square label_smoothing = 0 if loss_param is None else loss_param.label_smoothing @@ -183,7 +210,8 @@ def build(loss_type, pred, f1_beta_square, weights=loss_weight, - label_smoothing=label_smoothing) + label_smoothing=label_smoothing, + ) elif loss_type == LossType.BINARY_FOCAL_LOSS: if loss_param is None: return sigmoid_focal_loss_with_logits( @@ -200,7 +228,8 @@ def build(loss_type, ohem_ratio=loss_param.ohem_ratio, sample_weights=loss_weight, label_smoothing=loss_param.label_smoothing, - name=loss_name) + name=loss_name, + ) else: raise ValueError('unsupported loss type: %s' % LossType.Name(loss_type)) @@ -219,9 +248,10 @@ def build_kd_loss(kds, prediction_dict, label_dict, feature_dict): """ loss_dict = {} for kd in kds: - assert kd.pred_name in prediction_dict, \ - 'invalid predict_name: %s available ones: %s' % ( - kd.pred_name, ','.join(prediction_dict.keys())) + assert kd.pred_name in prediction_dict, 'invalid predict_name: %s available ones: %s' % ( + kd.pred_name, + ','.join(prediction_dict.keys()), + ) loss_name = kd.loss_name if not loss_name: @@ -232,8 +262,10 @@ def build_kd_loss(kds, prediction_dict, label_dict, feature_dict): if kd.HasField('task_space_indicator_name') and kd.HasField( 'task_space_indicator_value'): in_task_space = tf.to_float( - tf.equal(feature_dict[kd.task_space_indicator_name], - kd.task_space_indicator_value)) + tf.equal( + feature_dict[kd.task_space_indicator_name], + kd.task_space_indicator_value, + )) loss_weight = loss_weight * ( kd.in_task_space_weight * in_task_space + kd.out_task_space_weight * (1 - in_task_space)) @@ -329,5 +361,6 @@ def build_kd_loss(kds, prediction_dict, label_dict, feature_dict): pred, loss_weight=loss_weight, loss_param=loss_param, - **kwargs) + **kwargs, + ) return loss_dict diff --git a/easy_rec/python/builders/optimizer_builder.py b/easy_rec/python/builders/optimizer_builder.py index 8b9a8251b..2075e0811 100644 --- a/easy_rec/python/builders/optimizer_builder.py +++ b/easy_rec/python/builders/optimizer_builder.py @@ -14,6 +14,7 @@ # limitations under the License. # ============================================================================== """Functions to build training optimizers.""" + import logging import tensorflow as tf @@ -49,7 +50,8 @@ def build(optimizer_config): learning_rate, decay=config.decay, momentum=config.momentum_optimizer_value, - epsilon=config.epsilon) + epsilon=config.epsilon, + ) if optimizer_type == 'momentum_optimizer': config = optimizer_config.momentum_optimizer @@ -74,7 +76,8 @@ def build(optimizer_config): weight_decay=config.weight_decay, learning_rate=learning_rate, beta1=config.beta1, - beta2=config.beta2) + beta2=config.beta2, + ) if optimizer_type == 'adam_asyncw_optimizer': config = optimizer_config.adam_asyncw_optimizer @@ -86,13 +89,15 @@ def build(optimizer_config): weight_decay=config.weight_decay, learning_rate=learning_rate, beta1=config.beta1, - beta2=config.beta2) + beta2=config.beta2, + ) if optimizer_type == 'lazy_adam_optimizer': config = optimizer_config.lazy_adam_optimizer learning_rate = _create_learning_rate(config.learning_rate) summary_vars.append(learning_rate) from easy_rec.python.compat.adam_s import AdamOptimizerS + optimizer = AdamOptimizerS( learning_rate=learning_rate, beta1=config.beta1, beta2=config.beta2) @@ -105,7 +110,8 @@ def build(optimizer_config): optimizer = weight_decay_optimizers.MomentumWOptimizer( weight_decay=config.weight_decay, learning_rate=learning_rate, - momentum=config.momentum_optimizer_value) + momentum=config.momentum_optimizer_value, + ) if optimizer_type == 'adagrad_optimizer': config = optimizer_config.adagrad_optimizer @@ -132,7 +138,8 @@ def build(optimizer_config): initial_accumulator_value=config.initial_accumulator_value, l1_regularization_strength=config.l1_reg, l2_regularization_strength=config.l2_reg, - l2_shrinkage_regularization_strength=config.l2_shrinkage_reg) + l2_shrinkage_regularization_strength=config.l2_shrinkage_reg, + ) if optimizer is None: raise ValueError('Optimizer %s not supported.' % optimizer_type) @@ -173,7 +180,8 @@ def _create_learning_rate(learning_rate_config): burnin_learning_rate=config.burnin_learning_rate, burnin_steps=config.burnin_steps, min_learning_rate=config.min_learning_rate, - staircase=config.staircase) + staircase=config.staircase, + ) if learning_rate_type == 'manual_step_learning_rate': config = learning_rate_config.manual_step_learning_rate @@ -183,27 +191,42 @@ def _create_learning_rate(learning_rate_config): learning_rate_sequence = [config.initial_learning_rate] learning_rate_sequence += [x.learning_rate for x in config.schedule] learning_rate = learning_schedules.manual_stepping( - tf.train.get_or_create_global_step(), learning_rate_step_boundaries, - learning_rate_sequence, config.warmup) + tf.train.get_or_create_global_step(), + learning_rate_step_boundaries, + learning_rate_sequence, + config.warmup, + ) if learning_rate_type == 'cosine_decay_learning_rate': config = learning_rate_config.cosine_decay_learning_rate learning_rate = learning_schedules.cosine_decay_with_warmup( - tf.train.get_or_create_global_step(), config.learning_rate_base, - config.total_steps, config.warmup_learning_rate, config.warmup_steps, - config.hold_base_rate_steps) + tf.train.get_or_create_global_step(), + config.learning_rate_base, + config.total_steps, + config.warmup_learning_rate, + config.warmup_steps, + config.hold_base_rate_steps, + ) if learning_rate_type == 'poly_decay_learning_rate': config = learning_rate_config.poly_decay_learning_rate learning_rate = tf.train.polynomial_decay( - config.learning_rate_base, tf.train.get_or_create_global_step(), - config.total_steps, config.end_learning_rate, config.power) + config.learning_rate_base, + tf.train.get_or_create_global_step(), + config.total_steps, + config.end_learning_rate, + config.power, + ) if learning_rate_type == 'transformer_learning_rate': config = learning_rate_config.transformer_learning_rate learning_rate = learning_schedules.transformer_policy( - tf.train.get_or_create_global_step(), config.learning_rate_base, - config.hidden_size, config.warmup_steps, config.step_scaling_rate) + tf.train.get_or_create_global_step(), + config.learning_rate_base, + config.hidden_size, + config.warmup_steps, + config.step_scaling_rate, + ) if learning_rate is None: raise ValueError('Learning_rate %s not supported.' % learning_rate_type) diff --git a/easy_rec/python/builders/strategy_builder.py b/easy_rec/python/builders/strategy_builder.py index a0c758318..ef106423d 100644 --- a/easy_rec/python/builders/strategy_builder.py +++ b/easy_rec/python/builders/strategy_builder.py @@ -24,12 +24,14 @@ def build(train_config): # only works using pai-tf elif train_config.train_distribute == DistributionStrategy.ExascaleStrategy: import pai + distribution = pai.distribute.ExascaleStrategy( max_splits=10, issorted=True, optimize_clip_by_global_norm=False, enable_sparse_allreduce=False, - enable_hierarchical_allreduce=True) + enable_hierarchical_allreduce=True, + ) # the older version of MultiWorkerMirroredStrategy # works under tf1.12 to tf1.15 elif train_config.train_distribute == DistributionStrategy.CollectiveAllReduceStrategy: diff --git a/easy_rec/python/compat/adam_s.py b/easy_rec/python/compat/adam_s.py index a0ef60b80..909ea1bdd 100644 --- a/easy_rec/python/compat/adam_s.py +++ b/easy_rec/python/compat/adam_s.py @@ -13,20 +13,20 @@ # limitations under the License. # ============================================================================== """Adam for TensorFlow.""" + from __future__ import absolute_import from __future__ import division from __future__ import print_function from tensorflow.python.eager import context from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops from tensorflow.python.training import optimizer from tensorflow.python.training import training_ops +from tensorflow.python.ops import ( # NOQA + array_ops, control_flow_ops, math_ops, resource_variable_ops, state_ops, +) + class AdamOptimizerS(optimizer.Optimizer): """Optimizer that implements the Adam algorithm. @@ -37,13 +37,15 @@ class AdamOptimizerS(optimizer.Optimizer): ([pdf](https://arxiv.org/pdf/1412.6980.pdf)) """ - def __init__(self, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - use_locking=False, - name='Adam'): + def __init__( + self, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + use_locking=False, + name='Adam', + ): r"""Construct a new Adam optimizer. Initialization: @@ -118,8 +120,10 @@ def _get_beta_accumulators(self): graph = None else: graph = ops.get_default_graph() - return (self._get_non_slot_variable('beta1_power', graph=graph), - self._get_non_slot_variable('beta2_power', graph=graph)) + return ( + self._get_non_slot_variable('beta1_power', graph=graph), + self._get_non_slot_variable('beta2_power', graph=graph), + ) def _create_slots(self, var_list): # Create the beta1 and beta2 accumulators on the same device as the first @@ -163,7 +167,8 @@ def _apply_dense(self, grad, var): math_ops.cast(self._beta2_t, var.dtype.base_dtype), math_ops.cast(self._epsilon_t, var.dtype.base_dtype), grad, - use_locking=self._use_locking).op + use_locking=self._use_locking, + ).op def _resource_apply_dense(self, grad, var): m = self.get_slot(var, 'm') @@ -180,7 +185,8 @@ def _resource_apply_dense(self, grad, var): math_ops.cast(self._beta2_t, grad.dtype.base_dtype), math_ops.cast(self._epsilon_t, grad.dtype.base_dtype), grad, - use_locking=self._use_locking) + use_locking=self._use_locking, + ) def _apply_sparse_shared(self, grad, var, indices, scatter_add): beta1_power, beta2_power = self._get_beta_accumulators() @@ -190,7 +196,7 @@ def _apply_sparse_shared(self, grad, var, indices, scatter_add): beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype) beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype) epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype) - lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power)) + lr = lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power) # m_t = beta1 * m + (1 - beta1) * g_t m = self.get_slot(var, 'm') m_scaled_g_values = grad * (1 - beta1_t) @@ -221,7 +227,8 @@ def _apply_sparse(self, grad, var): x, i, v, - use_locking=self._use_locking)) + use_locking=self._use_locking), + ) def _resource_scatter_add(self, x, i, v): with ops.control_dependencies( diff --git a/easy_rec/python/compat/array_ops.py b/easy_rec/python/compat/array_ops.py index d788bc8c1..ea8bf5102 100644 --- a/easy_rec/python/compat/array_ops.py +++ b/easy_rec/python/compat/array_ops.py @@ -83,8 +83,8 @@ def _all_dimensions(x): # Fast path: avoid creating Rank and Range ops if ndims is known. if isinstance(x, ops.Tensor) and x.get_shape().ndims is not None: return constant_op.constant(np.arange(x.get_shape().ndims), dtype=tf.int32) - if (isinstance(x, sparse_tensor.SparseTensor) and - x.dense_shape.get_shape().is_fully_defined()): + if isinstance(x, sparse_tensor.SparseTensor) and x.dense_shape.get_shape( + ).is_fully_defined(): r = x.dense_shape.get_shape().dims[0].value # sparse.dense_shape is 1-D. return constant_op.constant(np.arange(r), dtype=tf.int32) diff --git a/easy_rec/python/compat/dynamic_variable.py b/easy_rec/python/compat/dynamic_variable.py index 83414331c..393ed7f52 100644 --- a/easy_rec/python/compat/dynamic_variable.py +++ b/easy_rec/python/compat/dynamic_variable.py @@ -23,8 +23,10 @@ from tensorflow.python.framework import ops # from tensorflow.python.ops import array_ops from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops.resource_variable_ops import ResourceVariable -from tensorflow.python.ops.resource_variable_ops import variable_accessed + +from tensorflow.python.ops.resource_variable_ops import ( # NOQA + ResourceVariable, variable_accessed, +) # from tensorflow.python.util import object_identity @@ -84,19 +86,21 @@ class DynamicVariable(ResourceVariable): print("v.size:", v.size) """ - def __init__(self, - dimension, - initializer=None, - var_type=None, - name=None, - constraint=None, - trainable=True, - key_type=None, - dtype=None, - mode=None, - variable_def=None, - import_scope=None, - **kwargs): + def __init__( + self, + dimension, + initializer=None, + var_type=None, + name=None, + constraint=None, + trainable=True, + key_type=None, + dtype=None, + mode=None, + variable_def=None, + import_scope=None, + **kwargs, + ): self._indices = None if variable_def is not None: super(DynamicVariable, self)._init_from_proto( @@ -105,7 +109,8 @@ def __init__(self, handle = g.as_graph_element( ops.prepend_name_scope( variable_def.variable_name, import_scope=import_scope), - allow_operation=False) + allow_operation=False, + ) self._dimension = handle.op.get_attr('shape').dim[-1].size self._key_type = handle.op.get_attr('key_type') self._handle_type = handle.op.get_attr('dtype') @@ -203,9 +208,8 @@ def __init__(self, self._initializer_op = tf.group([self._initializer_op, init_op]) # self._is_initialized_op = tf.group([self._is_initialized_op, is_initialized_op]) - handle_data = ( - resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult - .HandleData()) + handle_data = resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult.HandleData( + ) handle_data.is_set = True handle_data.shape_and_type.append( resource_variable_ops.cpp_shape_inference_pb2.CppShapeInferenceResult @@ -214,7 +218,8 @@ def __init__(self, resource_variable_ops._set_handle_shapes_and_types( self._handle, handle_data, - graph_mode=False if context.executing_eagerly() else True) + graph_mode=False if context.executing_eagerly() else True, + ) def is_static(self): return self._handle is self._tf_handle @@ -313,7 +318,8 @@ def sparse_read(self, indices, name=None, lookup_only=False): self._dummy_handle, indices, dtype=self._handle_dtype, - lookup_only=lookup_only) + lookup_only=lookup_only, + ) def scatter_sub(self, sparse_delta, use_locking=False, name=None): if self.is_static(): diff --git a/easy_rec/python/compat/early_stopping.py b/easy_rec/python/compat/early_stopping.py index fc850fb62..573a898c6 100644 --- a/easy_rec/python/compat/early_stopping.py +++ b/easy_rec/python/compat/early_stopping.py @@ -32,15 +32,17 @@ from tensorflow.python.platform import gfile from tensorflow.python.platform import tf_logging from tensorflow.python.summary import summary_iterator -from tensorflow.python.training import basic_session_run_hooks -from tensorflow.python.training import session_run_hook -from tensorflow.python.training import training_util from easy_rec.python.utils.config_util import parse_time from easy_rec.python.utils.load_class import load_by_path +from tensorflow.python.training import ( # NOQA + basic_session_run_hooks, session_run_hook, training_util, +) + if LooseVersion(tf.__version__) >= LooseVersion('2.12.0'): - from tensorflow_estimator.python.estimator.estimator_export import estimator_export + from tensorflow_estimator.python.estimator.estimator_export import ( # NOQA + estimator_export,) else: from tensorflow.python.util.tf_export import estimator_export @@ -111,13 +113,15 @@ def make_early_stopping_hook(estimator, @estimator_export('estimator.experimental.stop_if_higher_hook') -def stop_if_higher_hook(estimator, - metric_name, - threshold, - eval_dir=None, - min_steps=0, - run_every_secs=60, - run_every_steps=None): +def stop_if_higher_hook( + estimator, + metric_name, + threshold, + eval_dir=None, + min_steps=0, + run_every_secs=60, + run_every_steps=None, +): """Creates hook to stop if the given metric is higher than the threshold. Usage example: @@ -164,17 +168,20 @@ def stop_if_higher_hook(estimator, eval_dir=eval_dir, min_steps=min_steps, run_every_secs=run_every_secs, - run_every_steps=run_every_steps) + run_every_steps=run_every_steps, + ) @estimator_export('estimator.experimental.stop_if_lower_hook') -def stop_if_lower_hook(estimator, - metric_name, - threshold, - eval_dir=None, - min_steps=0, - run_every_secs=60, - run_every_steps=None): +def stop_if_lower_hook( + estimator, + metric_name, + threshold, + eval_dir=None, + min_steps=0, + run_every_secs=60, + run_every_steps=None, +): """Creates hook to stop if the given metric is lower than the threshold. Usage example: @@ -221,17 +228,20 @@ def stop_if_lower_hook(estimator, eval_dir=eval_dir, min_steps=min_steps, run_every_secs=run_every_secs, - run_every_steps=run_every_steps) + run_every_steps=run_every_steps, + ) @estimator_export('estimator.experimental.stop_if_no_increase_hook') -def stop_if_no_increase_hook(estimator, - metric_name, - max_steps_without_increase, - eval_dir=None, - min_steps=0, - run_every_secs=60, - run_every_steps=None): +def stop_if_no_increase_hook( + estimator, + metric_name, + max_steps_without_increase, + eval_dir=None, + min_steps=0, + run_every_secs=60, + run_every_steps=None, +): """Creates hook to stop if metric does not increase within given max steps. Usage example: @@ -279,15 +289,18 @@ def stop_if_no_increase_hook(estimator, eval_dir=eval_dir, min_steps=min_steps, run_every_secs=run_every_secs, - run_every_steps=run_every_steps) - - -def custom_early_stop_hook(estimator, - eval_dir, - custom_stop_func, - custom_stop_func_params, - run_every_secs=60, - run_every_steps=None): + run_every_steps=run_every_steps, + ) + + +def custom_early_stop_hook( + estimator, + eval_dir, + custom_stop_func, + custom_stop_func_params, + run_every_secs=60, + run_every_steps=None, +): """Custom early stop hook for complex early stop conditions. Args: @@ -311,7 +324,7 @@ def custom_early_stop_hook(estimator, eval_dir = estimator.eval_dir() if isinstance(custom_stop_func, str) or isinstance(custom_stop_func, - type(u'')): + type('')): custom_stop_func = load_by_path(custom_stop_func) def _custom_stop_fn(): @@ -322,17 +335,20 @@ def _custom_stop_fn(): estimator=estimator, should_stop_fn=_custom_stop_fn, run_every_secs=run_every_secs, - run_every_steps=run_every_steps) + run_every_steps=run_every_steps, + ) @estimator_export('estimator.experimental.stop_if_no_decrease_hook') -def stop_if_no_decrease_hook(estimator, - metric_name, - max_steps_without_decrease, - eval_dir=None, - min_steps=0, - run_every_secs=60, - run_every_steps=None): +def stop_if_no_decrease_hook( + estimator, + metric_name, + max_steps_without_decrease, + eval_dir=None, + min_steps=0, + run_every_secs=60, + run_every_steps=None, +): """Creates hook to stop if metric does not decrease within given max steps. Usage example: @@ -380,7 +396,8 @@ def stop_if_no_decrease_hook(estimator, eval_dir=eval_dir, min_steps=min_steps, run_every_secs=run_every_secs, - run_every_steps=run_every_steps) + run_every_steps=run_every_steps, + ) def read_eval_metrics(eval_dir): @@ -406,9 +423,16 @@ def read_eval_metrics(eval_dir): sorted(eval_metrics_dict.items(), key=lambda t: t[0])) -def _stop_if_threshold_crossed_hook(estimator, metric_name, threshold, - higher_is_better, eval_dir, min_steps, - run_every_secs, run_every_steps): +def _stop_if_threshold_crossed_hook( + estimator, + metric_name, + threshold, + higher_is_better, + eval_dir, + min_steps, + run_every_secs, + run_every_steps, +): """Creates early-stopping hook to stop training if threshold is crossed.""" if eval_dir is None: eval_dir = estimator.eval_dir() @@ -427,8 +451,13 @@ def stop_if_threshold_crossed_fn(): if is_lhs_better(val, threshold): tf_logging.info( 'At step %s, metric "%s" has value %s which is %s the configured ' - 'threshold (%s) for early stopping.', step, metric_name, val, - greater_or_lesser, threshold) + 'threshold (%s) for early stopping.', + step, + metric_name, + val, + greater_or_lesser, + threshold, + ) return True return False @@ -436,13 +465,20 @@ def stop_if_threshold_crossed_fn(): estimator=estimator, should_stop_fn=stop_if_threshold_crossed_fn, run_every_secs=run_every_secs, - run_every_steps=run_every_steps) - - -def _stop_if_no_metric_improvement_hook(estimator, metric_name, - max_steps_without_improvement, - higher_is_better, eval_dir, min_steps, - run_every_secs, run_every_steps): + run_every_steps=run_every_steps, + ) + + +def _stop_if_no_metric_improvement_hook( + estimator, + metric_name, + max_steps_without_improvement, + higher_is_better, + eval_dir, + min_steps, + run_every_secs, + run_every_steps, +): """Returns hook to stop training if given metric shows no improvement.""" if eval_dir is None: eval_dir = estimator.eval_dir() @@ -467,8 +503,11 @@ def stop_if_no_metric_improvement_fn(): tf_logging.info( 'No %s in metric "%s" for %s steps, which is greater than or equal ' 'to max steps (%s) configured for early stopping.', - increase_or_decrease, metric_name, step - best_val_step, - max_steps_without_improvement) + increase_or_decrease, + metric_name, + step - best_val_step, + max_steps_without_improvement, + ) return True return False @@ -476,7 +515,8 @@ def stop_if_no_metric_improvement_fn(): estimator=estimator, should_stop_fn=stop_if_no_metric_improvement_fn, run_every_secs=run_every_secs, - run_every_steps=run_every_steps) + run_every_steps=run_every_steps, + ) def _summaries(eval_dir): @@ -506,7 +546,8 @@ def _get_or_create_stop_var(): dtype=dtypes.bool, initializer=init_ops.constant_initializer(False), collections=[ops.GraphKeys.GLOBAL_VARIABLES], - trainable=False) + trainable=False, + ) class _StopOnPredicateHook(session_run_hook.SessionRunHook): @@ -619,7 +660,8 @@ def oss_stop_hook(estimator, run_every_secs=10, run_every_steps=None): return OssStopSignalHook( estimator.model_dir, run_every_secs=run_every_secs, - run_every_steps=run_every_steps) + run_every_steps=run_every_steps, + ) else: return _CheckForStoppingHook() diff --git a/easy_rec/python/compat/embedding_ops.py b/easy_rec/python/compat/embedding_ops.py index 2ad3af7f9..b1ffd8080 100644 --- a/easy_rec/python/compat/embedding_ops.py +++ b/easy_rec/python/compat/embedding_ops.py @@ -2,14 +2,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. """Add embedding column for EmbeddingVariable which is only available on pai.""" -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.framework import sparse_tensor -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import sparse_ops +from tensorflow.python.framework import dtypes, ops, sparse_tensor, tensor_shape # NOQA +from tensorflow.python.ops import array_ops, embedding_ops, math_ops, sparse_ops # NOQA def _prune_invalid_ids(sparse_ids, sparse_weights): @@ -34,14 +28,16 @@ def _prune_invalid_weights(sparse_ids, sparse_weights): return sparse_ids, sparse_weights -def safe_embedding_lookup_sparse(embedding_weights, - sparse_ids, - sparse_weights=None, - combiner='mean', - default_id=None, - name=None, - partition_strategy='div', - max_norm=None): +def safe_embedding_lookup_sparse( + embedding_weights, + sparse_ids, + sparse_weights=None, + combiner='mean', + default_id=None, + name=None, + partition_strategy='div', + max_norm=None, +): """Lookup embedding results, accounting for invalid IDs and empty features. Fixed so that could be used with Pai EmbeddingVariables. @@ -96,14 +92,17 @@ def safe_embedding_lookup_sparse(embedding_weights, # Reshape higher-rank sparse ids and weights to linear segment ids. original_shape = sparse_ids.dense_shape original_rank_dim = sparse_ids.dense_shape.get_shape()[0] - original_rank = ( - array_ops.size(original_shape) - if original_rank_dim.value is None else original_rank_dim.value) - sparse_ids = sparse_ops.sparse_reshape(sparse_ids, [ - math_ops.reduce_prod( - array_ops.slice(original_shape, [0], [original_rank - 1])), - array_ops.gather(original_shape, original_rank - 1) - ]) + original_rank = array_ops.size( + original_shape + ) if original_rank_dim.value is None else original_rank_dim.value + sparse_ids = sparse_ops.sparse_reshape( + sparse_ids, + [ + math_ops.reduce_prod( + array_ops.slice(original_shape, [0], [original_rank - 1])), + array_ops.gather(original_shape, original_rank - 1), + ], + ) if sparse_weights is not None: sparse_weights = sparse_tensor.SparseTensor(sparse_ids.indices, sparse_weights.values, @@ -135,14 +134,16 @@ def safe_embedding_lookup_sparse(embedding_weights, combiner=combiner, partition_strategy=partition_strategy, name=None if default_id is None else scope, - max_norm=max_norm) + max_norm=max_norm, + ) if default_id is None: # Broadcast is_row_empty to the same shape as embedding_lookup_result, # for use in Select. is_row_empty = array_ops.tile( array_ops.reshape(is_row_empty, [-1, 1]), - array_ops.stack([1, array_ops.shape(result)[1]])) + array_ops.stack([1, array_ops.shape(result)[1]]), + ) result = array_ops.where( is_row_empty, array_ops.zeros_like(result), result, name=scope) @@ -150,12 +151,18 @@ def safe_embedding_lookup_sparse(embedding_weights, # Reshape back from linear ids back into higher-dimensional dense result. final_result = array_ops.reshape( result, - array_ops.concat([ - array_ops.slice( - math_ops.cast(original_shape, dtypes.int32), [0], - [original_rank - 1]), - array_ops.slice(array_ops.shape(result), [1], [-1]) - ], 0)) + array_ops.concat( + [ + array_ops.slice( + math_ops.cast(original_shape, dtypes.int32), + [0], + [original_rank - 1], + ), + array_ops.slice(array_ops.shape(result), [1], [-1]), + ], + 0, + ), + ) final_result.set_shape( tensor_shape.unknown_shape( (original_rank_dim - 1).value).concatenate(result.get_shape()[1:])) diff --git a/easy_rec/python/compat/embedding_parallel_saver.py b/easy_rec/python/compat/embedding_parallel_saver.py index 96c08c592..66a9610b0 100644 --- a/easy_rec/python/compat/embedding_parallel_saver.py +++ b/easy_rec/python/compat/embedding_parallel_saver.py @@ -7,20 +7,21 @@ from tensorflow.core.protobuf import saver_pb2 from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -# from tensorflow.python.ops import math_ops -# from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import script_ops -from tensorflow.python.ops import state_ops from tensorflow.python.platform import gfile from tensorflow.python.training import saver from easy_rec.python.utils import constant +# from tensorflow.python.ops import math_ops +# from tensorflow.python.ops import logging_ops +from tensorflow.python.ops import ( # NOQA + array_ops, control_flow_ops, script_ops, state_ops, +) + try: import horovod.tensorflow as hvd from sparse_operation_kit.experiment import raw_ops as dynamic_variable_ops + from easy_rec.python.compat import dynamic_variable except Exception: dynamic_variable_ops = None @@ -28,7 +29,9 @@ try: from tensorflow.python.framework.load_library import load_op_library + import easy_rec + load_embed_lib_path = os.path.join(easy_rec.ops_dir, 'libload_embed.so') load_embed_lib = load_op_library(load_embed_lib_path) except Exception as ex: @@ -45,22 +48,24 @@ def _get_embed_part_id(embed_file): class EmbeddingParallelSaver(saver.Saver): - def __init__(self, - var_list=None, - reshape=False, - sharded=False, - max_to_keep=5, - keep_checkpoint_every_n_hours=10000.0, - name=None, - restore_sequentially=False, - saver_def=None, - builder=None, - defer_build=False, - allow_empty=False, - write_version=saver_pb2.SaverDef.V2, - pad_step_number=False, - save_relative_paths=False, - filename=None): + def __init__( + self, + var_list=None, + reshape=False, + sharded=False, + max_to_keep=5, + keep_checkpoint_every_n_hours=10000.0, + name=None, + restore_sequentially=False, + saver_def=None, + builder=None, + defer_build=False, + allow_empty=False, + write_version=saver_pb2.SaverDef.V2, + pad_step_number=False, + save_relative_paths=False, + filename=None, + ): self._kv_vars = [] self._embed_vars = [] tf_vars = [] @@ -90,7 +95,8 @@ def __init__(self, write_version=write_version, pad_step_number=pad_step_number, save_relative_paths=save_relative_paths, - filename=filename) + filename=filename, + ) self._is_build = False def _has_embed_vars(self): @@ -157,8 +163,10 @@ def _load_embed(embed, embed_dim, embed_part_size, part_id, part_num, embed_ids_o = np.arange(len(embed_val)) embed_ids_o = part_id_o + embed_ids_o * len(embed_files) sel_ids = np.where( - np.logical_and((embed_ids_o % part_num) == part_id, - embed_ids_o < embed_part_size * part_num))[0] + np.logical_and( + (embed_ids_o % part_num) == part_id, + embed_ids_o < embed_part_size * part_num, + ))[0] part_update_cnt += len(sel_ids) embed_ids = embed_ids_o[sel_ids] embed_ids_n = np.array(embed_ids / part_num, dtype=np.int64) @@ -174,13 +182,22 @@ def _load_embed(embed, embed_dim, embed_part_size, part_id, part_num, embed_dim=embed_dim, embed_part_size=embed_part_size, var_name='embed-' + embed_var.name.replace('/', '__'), - ckpt_path=file_name) + ckpt_path=file_name, + ) else: - embed_val = script_ops.py_func(_load_embed, [ - embed_var, embed_dim, embed_part_size, - hvd.rank(), - hvd.size(), file_name, embed_var.name - ], dtypes.float32) + embed_val = script_ops.py_func( + _load_embed, + [ + embed_var, + embed_dim, + embed_part_size, + hvd.rank(), + hvd.size(), + file_name, + embed_var.name, + ], + dtypes.float32, + ) embed_val.set_shape(embed_var.get_shape()) return state_ops.assign(embed_var, embed_val) @@ -243,16 +260,22 @@ def _load_key_vals(filename, var_name): if len(tmp_ids) == 0: break all_keys.append(tmp_keys.take(tmp_ids, axis=0)) - logging.info('part_keys.shape=%s %s %s' % (str( - tmp_keys.shape), str(tmp_ids.shape), str(all_keys[-1].shape))) + logging.info('part_keys.shape=%s %s %s' % ( + str(tmp_keys.shape), + str(tmp_ids.shape), + str(all_keys[-1].shape), + )) val_file = key_file[:-4] + 'vals' with gfile.GFile(val_file, 'rb') as fin: tmp_vals = np.frombuffer( fin.read(), dtype=np.float32).reshape([-1, sok_var._dimension]) all_vals.append(tmp_vals.take(tmp_ids, axis=0)) - logging.info('part_vals.shape=%s %s %s' % (str( - tmp_vals.shape), str(tmp_ids.shape), str(all_vals[-1].shape))) + logging.info('part_vals.shape=%s %s %s' % ( + str(tmp_vals.shape), + str(tmp_ids.shape), + str(all_vals[-1].shape), + )) all_keys = np.concatenate(all_keys, axis=0) all_vals = np.concatenate(all_vals, axis=0) @@ -271,11 +294,15 @@ def _load_key_vals(filename, var_name): task_num=hvd.size(), embed_dim=sok_var._dimension, var_name='embed-' + sok_var.name.replace('/', '__'), - ckpt_path=file_name) + ckpt_path=file_name, + ) else: logging.warning('libload_embed.so not loaded, will use python script_ops') - keys, vals = script_ops.py_func(_load_key_vals, [file_name, sok_var.name], - (dtypes.int64, dtypes.float32)) + keys, vals = script_ops.py_func( + _load_key_vals, + [file_name, sok_var.name], + (dtypes.int64, dtypes.float32), + ) with ops.control_dependencies([sok_var._initializer_op]): return dynamic_variable_ops.dummy_var_assign(sok_var.handle, keys, vals) diff --git a/easy_rec/python/compat/estimator_train.py b/easy_rec/python/compat/estimator_train.py index 1ec71e491..dbd853759 100644 --- a/easy_rec/python/compat/estimator_train.py +++ b/easy_rec/python/compat/estimator_train.py @@ -5,15 +5,15 @@ import tensorflow as tf from tensorflow.python.estimator import run_config as run_config_lib -from tensorflow.python.estimator.training import _assert_eval_spec -from tensorflow.python.estimator.training import _ContinuousEvalListener -from tensorflow.python.estimator.training import _TrainingExecutor from tensorflow.python.util import compat from easy_rec.python.compat.exporter import FinalExporter from easy_rec.python.utils import estimator_utils from tensorflow.python.distribute import estimator_training as distribute_coordinator_training # NOQA +from tensorflow.python.estimator.training import ( # NOQA + _assert_eval_spec, _ContinuousEvalListener, _TrainingExecutor, +) if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -63,7 +63,8 @@ def train_and_evaluate(estimator, train_spec, eval_spec): estimator=estimator, train_spec=train_spec, eval_spec=eval_spec, - continuous_eval_listener=train_done_listener) + continuous_eval_listener=train_done_listener, + ) config = estimator.config # If `distribute_coordinator_mode` is set and running in distributed @@ -75,8 +76,7 @@ def train_and_evaluate(estimator, train_spec, eval_spec): _TrainingExecutor) return - if (config.task_type == run_config_lib.TaskType.EVALUATOR and - config.task_id > 0): + if config.task_type == run_config_lib.TaskType.EVALUATOR and config.task_id > 0: raise ValueError( 'For distributed training, there can only be one `evaluator` task ' '(with task id 0). Given task id {}'.format(config.task_id)) @@ -102,7 +102,8 @@ def train_and_evaluate(estimator, train_spec, eval_spec): checkpoint_path=estimator_utils.latest_checkpoint( estimator.model_dir), eval_result=None, - is_the_final_export=True) + is_the_final_export=True, + ) if estimator_utils.is_chief(): with gfile.GFile(train_done_listener.train_done_file, 'w') as fout: diff --git a/easy_rec/python/compat/exporter.py b/easy_rec/python/compat/exporter.py index d8e3ed418..5fa43e431 100644 --- a/easy_rec/python/compat/exporter.py +++ b/easy_rec/python/compat/exporter.py @@ -92,14 +92,16 @@ class BestExporter(Exporter): existing model. """ - def __init__(self, - name='best_exporter', - serving_input_receiver_fn=None, - event_file_pattern='eval_val/*.tfevents.*', - compare_fn=_loss_smaller, - assets_extra=None, - as_text=False, - exports_to_keep=5): + def __init__( + self, + name='best_exporter', + serving_input_receiver_fn=None, + event_file_pattern='eval_val/*.tfevents.*', + compare_fn=_loss_smaller, + assets_extra=None, + as_text=False, + exports_to_keep=5, + ): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. Example of creating a BestExporter for training and evaluation: @@ -217,10 +219,13 @@ def export(self, estimator, export_path, checkpoint_path, eval_result, current_eval_result=eval_result): tf_logging.info('Performing best model export.') self._best_eval_result = eval_result - export_result = self._saved_model_exporter.export(estimator, export_path, - checkpoint_path, - eval_result, - is_the_final_export) + export_result = self._saved_model_exporter.export( + estimator, + export_path, + checkpoint_path, + eval_result, + is_the_final_export, + ) self._garbage_collect_exports(export_path) # cp best checkpoints to best folder model_dir, _ = os.path.split(checkpoint_path) @@ -390,12 +395,14 @@ class LatestExporter(Exporter): In addition to exporting, this class also garbage collects stale exports. """ - def __init__(self, - name, - serving_input_receiver_fn, - assets_extra=None, - as_text=False, - exports_to_keep=5): + def __init__( + self, + name, + serving_input_receiver_fn, + assets_extra=None, + as_text=False, + exports_to_keep=5, + ): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. Args: diff --git a/easy_rec/python/compat/feature_column/feature_column.py b/easy_rec/python/compat/feature_column/feature_column.py index 73a568d9c..308f6f581 100644 --- a/easy_rec/python/compat/feature_column/feature_column.py +++ b/easy_rec/python/compat/feature_column/feature_column.py @@ -146,23 +146,6 @@ from tensorflow.python.framework import tensor_shape from tensorflow.python.keras.engine import training from tensorflow.python.layers import base -# from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import string_ops -from tensorflow.python.ops import template -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables # from tensorflow.python.ops.ragged import ragged_tensor # from tensorflow.python.ops.ragged import ragged_util from tensorflow.python.platform import gfile @@ -175,6 +158,13 @@ from easy_rec.python.utils import constant from easy_rec.python.utils import embedding_utils +# from tensorflow.python.ops import logging_ops +from tensorflow.python.ops import ( # NOQA + array_ops, check_ops, control_flow_ops, data_flow_ops, embedding_ops, + init_ops, lookup_ops, math_ops, nn_ops, parsing_ops, resource_variable_ops, + sparse_ops, string_ops, template, variable_scope, variables, +) + try: from easy_rec.python.compat import dynamic_variable except Exception: @@ -245,12 +235,14 @@ def embedding_lookup_ragged(embedding_weights, # model parallel embedding lookup -def embedding_parallel_lookup(embedding, - lookup_indices, - output_ids, - is_training, - output_tensors=None, - batch_size=None): +def embedding_parallel_lookup( + embedding, + lookup_indices, + output_ids, + is_training, + output_tensors=None, + batch_size=None, +): N = len(output_ids) if batch_size is None: num_segments = None @@ -264,10 +256,13 @@ def embedding_parallel_lookup(embedding, cumsum_lens = math_ops.cumsum(segment_lens) segment_ids = array_ops.searchsorted( cumsum_lens, math_ops.range(cumsum_lens[-1]), side='right') - elif isinstance(lookup_indices, dict) and 'ragged_ids' in lookup_indices.keys( - ) and 'ragged_lens' in lookup_indices.keys(): - all_ids, segment_lens = lookup_indices['ragged_ids'], lookup_indices[ - 'ragged_lens'] + elif (isinstance(lookup_indices, dict) and + 'ragged_ids' in lookup_indices.keys() and + 'ragged_lens' in lookup_indices.keys()): + all_ids, segment_lens = ( + lookup_indices['ragged_ids'], + lookup_indices['ragged_lens'], + ) all_uniq_ids, uniq_idx = array_ops.unique(all_ids) cumsum_lens = math_ops.cumsum(segment_lens) segment_ids = array_ops.searchsorted( @@ -328,7 +323,8 @@ def embedding_parallel_lookup(embedding, uniq_idx, segment_ids, num_segments=num_segments, - name='sparse_segment_sum') + name='sparse_segment_sum', + ) else: if isinstance(embedding, dynamic_variable.DynamicVariable): recv_embeddings = embedding.sparse_read( @@ -340,7 +336,8 @@ def embedding_parallel_lookup(embedding, uniq_idx, segment_ids, num_segments=num_segments, - name='sparse_segment_sum') + name='sparse_segment_sum', + ) embed_dim = embedding.get_shape()[-1] output_tensor = array_ops.reshape(embeddings, [N, -1, embed_dim]) @@ -357,16 +354,18 @@ def embedding_parallel_lookup(embedding, [batch_size, N * embed_dim]) -def _internal_input_layer(features, - feature_columns, - weight_collections=None, - trainable=True, - cols_to_vars=None, - scope=None, - cols_to_output_tensors=None, - from_template=False, - feature_name_to_output_tensors=None, - is_training=True): +def _internal_input_layer( + features, + feature_columns, + weight_collections=None, + trainable=True, + cols_to_vars=None, + scope=None, + cols_to_output_tensors=None, + from_template=False, + feature_name_to_output_tensors=None, + is_training=True, +): """See input_layer, `scope` is a name or variable scope to use.""" feature_columns = _normalize_feature_columns(feature_columns) for column in feature_columns: @@ -381,7 +380,7 @@ def _internal_input_layer(features, if ops.GraphKeys.MODEL_VARIABLES not in weight_collections: weight_collections.append(ops.GraphKeys.MODEL_VARIABLES) - def _get_logits(): # pylint: disable=missing-docstring + def _get_logits(): builder = _LazyBuilder(features) output_tensors = [] @@ -391,12 +390,10 @@ def _get_logits(): # pylint: disable=missing-docstring tmp_cols = sorted(tmp_cols, key=lambda x: x.name) for column in tmp_cols: with variable_scope.variable_scope( - None, default_name=column._var_scope_name): # pylint: disable=protected-access - tensor = column._get_dense_tensor( # pylint: disable=protected-access - builder, - weight_collections=weight_collections, - trainable=trainable) - num_elements = column._variable_shape.num_elements() # pylint: disable=protected-access + None, default_name=column._var_scope_name): + tensor = column._get_dense_tensor( + builder, weight_collections=weight_collections, trainable=trainable) + num_elements = column._variable_shape.num_elements() batch_size = array_ops.shape(tensor)[0] output_tensor = array_ops.reshape( tensor, shape=(batch_size, num_elements)) @@ -406,14 +403,15 @@ def _get_logits(): # pylint: disable=missing-docstring # variables, in which case an empty list is returned). cols_to_vars[column] = ops.get_collection( ops.GraphKeys.GLOBAL_VARIABLES, - scope=variable_scope.get_variable_scope().name) + scope=variable_scope.get_variable_scope().name, + ) if cols_to_output_tensors is not None: cols_to_output_tensors[column] = output_tensor if feature_name_to_output_tensors is not None: feature_name_to_output_tensors[column.raw_name] = output_tensor return array_ops.concat(output_tensors, 1) - def _get_logits_embedding_parallel(): # pylint: disable=missing-docstring + def _get_logits_embedding_parallel(): assert hvd is not None, 'horovod is not installed' builder = _LazyBuilder(features) @@ -448,7 +446,7 @@ def _get_var_type(column): for column in feature_columns: ordered_columns.append(column) with variable_scope.variable_scope( - None, default_name=column._var_scope_name): # pylint: disable=protected-access + None, default_name=column._var_scope_name): # for features which does not require embedding if 'Embedding' not in str(type(column)): dense_cols.append(column) @@ -477,7 +475,8 @@ def _get_var_type(column): trainable=column.trainable and trainable, dtype=dtypes.float32, init_capacity=column.ev_params.init_capacity, - max_capacity=column.ev_params.max_capacity) + max_capacity=column.ev_params.max_capacity, + ) else: embedding_weights = variable_scope.get_variable( name='embedding_weights', @@ -486,7 +485,8 @@ def _get_var_type(column): initializer=column.initializer, trainable=column.trainable and trainable, partitioner=None, - collections=weight_collections) + collections=weight_collections, + ) shared_weights[shared_name] = embedding_weights else: with ops.device(embedding_device): @@ -500,7 +500,8 @@ def _get_var_type(column): trainable=column.trainable and trainable, dtype=dtypes.float32, init_capacity=column.ev_params.init_capacity, - max_capacity=column.ev_params.max_capacity) + max_capacity=column.ev_params.max_capacity, + ) else: embedding_weights = variable_scope.get_variable( name='embedding_weights', @@ -509,7 +510,8 @@ def _get_var_type(column): initializer=column.initializer, trainable=column.trainable and trainable, partitioner=None, - collections=weight_collections) + collections=weight_collections, + ) lookup_embeddings.append(embedding_weights) output_id = len(output_tensors) output_tensors.append(None) @@ -527,7 +529,7 @@ def _get_var_type(column): if lookup_indices is None: lookup_indices = { 'ragged_ids': features['ragged_ids'], - 'ragged_lens': features['ragged_lens'] + 'ragged_lens': features['ragged_lens'], } if 'ragged_wgts' in features: lookup_indices['ragged_wgts'] = features['ragged_wgts'] @@ -538,14 +540,16 @@ def _get_var_type(column): sparse_tensors = column.categorical_column._get_sparse_tensors( builder, weight_collections=weight_collections, - trainable=trainable) + trainable=trainable, + ) lookup_indices.append(sparse_tensors.id_tensor) if sparse_tensors.weight_tensor is not None: lookup_wgts.append(sparse_tensors.weight_tensor) if cols_to_vars is not None: cols_to_vars[column] = ops.get_collection( ops.GraphKeys.GLOBAL_VARIABLES, - scope=variable_scope.get_variable_scope().name) + scope=variable_scope.get_variable_scope().name, + ) if dense_cnt > 0: if 'dense_fea' in features: @@ -570,14 +574,18 @@ def _get_var_type(column): batch_size = batch_sizes[0] # do embedding parallel lookup if len(lookup_output_ids) > 0: - packed_input = ('sparse_fea' in features or 'ragged_ids' in features) + packed_input = 'sparse_fea' in features or 'ragged_ids' in features if packed_input: uniq_embed_cnt = len(set(lookup_embeddings)) assert uniq_embed_cnt == 1, 'only one uniq embed is support for packed inputs' - outputs = embedding_parallel_lookup(lookup_embeddings[0], - lookup_indices, lookup_output_ids, - is_training, output_tensors, - batch_size) + outputs = embedding_parallel_lookup( + lookup_embeddings[0], + lookup_indices, + lookup_output_ids, + is_training, + output_tensors, + batch_size, + ) else: if batch_size is None: all_indices = [] @@ -593,7 +601,7 @@ def _get_var_type(column): if embedding not in grouped_inputs: grouped_inputs[embedding] = { 'lookup_indice': [lookup_indice], - 'output_id': [output_id] + 'output_id': [output_id], } else: grouped_inputs[embedding]['lookup_indice'].append(lookup_indice) @@ -602,9 +610,14 @@ def _get_var_type(column): for embedding in grouped_inputs: lookup_indices = grouped_inputs[embedding]['lookup_indice'] output_ids = grouped_inputs[embedding]['output_id'] - outputs = embedding_parallel_lookup(embedding, lookup_indices, - output_ids, is_training, - output_tensors, batch_size) + outputs = embedding_parallel_lookup( + embedding, + lookup_indices, + output_ids, + is_training, + output_tensors, + batch_size, + ) for output_tensor, col in zip(output_tensors, feature_columns): if feature_name_to_output_tensors is not None: @@ -640,14 +653,16 @@ def _get_var_type(column): return _get_logits() -def input_layer(features, - feature_columns, - weight_collections=None, - trainable=True, - cols_to_vars=None, - cols_to_output_tensors=None, - feature_name_to_output_tensors=None, - is_training=True): +def input_layer( + features, + feature_columns, + weight_collections=None, + trainable=True, + cols_to_vars=None, + cols_to_output_tensors=None, + feature_name_to_output_tensors=None, + is_training=True, +): """Returns a dense `Tensor` as input layer based on given `feature_columns`. Generally a single example in training data is described with FeatureColumns. @@ -712,7 +727,8 @@ def input_layer(features, cols_to_vars=cols_to_vars, cols_to_output_tensors=cols_to_output_tensors, feature_name_to_output_tensors=feature_name_to_output_tensors, - is_training=is_training) + is_training=is_training, + ) # TODO(akshayka): InputLayer should be a subclass of Layer, and it @@ -722,13 +738,15 @@ def input_layer(features, class InputLayer(object): """An object-oriented version of `input_layer` that reuses variables.""" - def __init__(self, - feature_columns, - weight_collections=None, - trainable=True, - cols_to_vars=None, - name='feature_column_input_layer', - create_scope_now=True): + def __init__( + self, + feature_columns, + weight_collections=None, + trainable=True, + cols_to_vars=None, + name='feature_column_input_layer', + create_scope_now=True, + ): """See `input_layer`.""" self._feature_columns = feature_columns self._weight_collections = weight_collections @@ -746,7 +764,8 @@ def __call__(self, features): weight_collections=self._weight_collections, trainable=self._trainable, cols_to_vars=None, - from_template=True) + from_template=True, + ) @property def name(self): @@ -777,13 +796,15 @@ def weights(self): return self._input_layer_template.weights -def linear_model(features, - feature_columns, - units=1, - sparse_combiner='sum', - weight_collections=None, - trainable=True, - cols_to_vars=None): +def linear_model( + features, + feature_columns, + units=1, + sparse_combiner='sum', + weight_collections=None, + trainable=True, + cols_to_vars=None, +): """Returns a linear prediction `Tensor` based on given `feature_columns`. This function generates a weighted sum based on output dimension `units`. @@ -907,8 +928,9 @@ def linear_model(features, sparse_combiner=sparse_combiner, weight_collections=weight_collections, trainable=trainable, - name=model_name) - retval = linear_model_layer(features) # pylint: disable=not-callable + name=model_name, + ) + retval = linear_model_layer(features) if cols_to_vars is not None: cols_to_vars.update(linear_model_layer.cols_to_vars()) return retval @@ -942,14 +964,16 @@ class _FCLinearWrapper(base.Layer): See `linear_model` above. """ - def __init__(self, - feature_column, - units=1, - sparse_combiner='sum', - weight_collections=None, - trainable=True, - name=None, - **kwargs): + def __init__( + self, + feature_column, + units=1, + sparse_combiner='sum', + weight_collections=None, + trainable=True, + name=None, + **kwargs, + ): super(_FCLinearWrapper, self).__init__( trainable=trainable, name=name, **kwargs) self._feature_column = feature_column @@ -961,16 +985,18 @@ def build(self, _): if isinstance(self._feature_column, _CategoricalColumn): weight = self.add_variable( name='weights', - shape=(self._feature_column._num_buckets, self._units), # pylint: disable=protected-access + shape=(self._feature_column._num_buckets, self._units), initializer=init_ops.zeros_initializer(), - trainable=self.trainable) + trainable=self.trainable, + ) else: - num_elements = self._feature_column._variable_shape.num_elements() # pylint: disable=protected-access + num_elements = self._feature_column._variable_shape.num_elements() weight = self.add_variable( name='weights', shape=[num_elements, self._units], initializer=init_ops.zeros_initializer(), - trainable=self.trainable) + trainable=self.trainable, + ) _add_to_collections(weight, self._weight_collections) self._weight_var = weight self.built = True @@ -983,7 +1009,8 @@ def call(self, builder): sparse_combiner=self._sparse_combiner, weight_collections=self._weight_collections, trainable=self.trainable, - weight_var=self._weight_var) + weight_var=self._weight_var, + ) return weighted_sum @@ -1005,7 +1032,8 @@ def build(self, _): 'bias_weights', shape=[self._units], initializer=init_ops.zeros_initializer(), - trainable=self.trainable) + trainable=self.trainable, + ) _add_to_collections(self._bias_variable, self._weight_collections) self.built = True @@ -1014,8 +1042,8 @@ def call(self, _): def _get_expanded_variable_list(variable): - if (isinstance(variable, variables.Variable) or - resource_variable_ops.is_resource_variable(variable)): + if isinstance(variable, variables.Variable + ) or resource_variable_ops.is_resource_variable(variable): return [variable] # Single variable case. else: # Must be a PartitionedVariable, so convert into a list. return list(variable) @@ -1031,14 +1059,16 @@ class _LinearModel(training.Model): See `linear_model` for details. """ - def __init__(self, - feature_columns, - units=1, - sparse_combiner='sum', - weight_collections=None, - trainable=True, - name=None, - **kwargs): + def __init__( + self, + feature_columns, + units=1, + sparse_combiner='sum', + weight_collections=None, + trainable=True, + name=None, + **kwargs, + ): super(_LinearModel, self).__init__(name=name, **kwargs) self._feature_columns = _normalize_feature_columns(feature_columns) self._weight_collections = list(weight_collections or []) @@ -1050,14 +1080,20 @@ def __init__(self, column_layers = {} for column in sorted(self._feature_columns, key=lambda x: x.name): with variable_scope.variable_scope( - None, default_name=column._var_scope_name) as vs: # pylint: disable=protected-access + None, default_name=column._var_scope_name) as vs: # Having the fully expressed variable scope name ends up doubly # expressing the outer scope (scope with which this method was called) # in the name of the variable that would get created. column_name = _strip_leading_slashes(vs.name) - column_layer = _FCLinearWrapper(column, units, sparse_combiner, - self._weight_collections, trainable, - column_name, **kwargs) + column_layer = _FCLinearWrapper( + column, + units, + sparse_combiner, + self._weight_collections, + trainable, + column_name, + **kwargs, + ) column_layers[column_name] = column_layer self._column_layers = self._add_layers(column_layers) self._bias_layer = _BiasLayer( @@ -1065,7 +1101,8 @@ def __init__(self, trainable=trainable, weight_collections=self._weight_collections, name='bias_layer', - **kwargs) + **kwargs, + ) self._cols_to_vars = {} def cols_to_vars(self): @@ -1087,7 +1124,7 @@ def call(self, features): ordered_columns = [] builder = _LazyBuilder(features) for layer in sorted(self._column_layers.values(), key=lambda x: x.name): - column = layer._feature_column # pylint: disable=protected-access + column = layer._feature_column ordered_columns.append(column) weighted_sum = layer(builder) weighted_sums.append(weighted_sum) @@ -1099,10 +1136,9 @@ def call(self, features): weighted_sums, name='weighted_sum_no_bias') predictions = nn_ops.bias_add( predictions_no_bias, - self._bias_layer( # pylint: disable=not-callable - builder, - scope=variable_scope.get_variable_scope()), # pylint: disable=not-callable - name='weighted_sum') + self._bias_layer(builder, scope=variable_scope.get_variable_scope()), + name='weighted_sum', + ) bias = self._bias_layer.variables[0] self._cols_to_vars['bias'] = _get_expanded_variable_list(bias) return predictions @@ -1210,7 +1246,7 @@ def make_parse_example_spec(feature_columns): if not isinstance(column, _FeatureColumn): raise ValueError('All feature_columns must be _FeatureColumn instances. ' 'Given: {}'.format(column)) - config = column._parse_example_spec # pylint: disable=protected-access + config = column._parse_example_spec for key, value in six.iteritems(config): if key in result and value != result[key]: raise ValueError('feature_columns contain different parse_spec for key ' @@ -1219,14 +1255,16 @@ def make_parse_example_spec(feature_columns): return result -def _embedding_column(categorical_column, - dimension, - combiner='mean', - initializer=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - max_norm=None, - trainable=True): +def _embedding_column( + categorical_column, + dimension, + combiner='mean', + initializer=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True, +): """`_DenseColumn` that converts from sparse, categorical input. Use this when your inputs are sparse, but you want to convert them to a dense @@ -1312,7 +1350,7 @@ def model_fn(features, ...): initializer = init_ops.truncated_normal_initializer( mean=0.0, stddev=0.01 / math.sqrt(dimension)) - embedding_shape = categorical_column._num_buckets, dimension # pylint: disable=protected-access + embedding_shape = categorical_column._num_buckets, dimension def _creator(weight_collections, scope): embedding_column_layer = _EmbeddingColumnLayer( @@ -1320,8 +1358,9 @@ def _creator(weight_collections, scope): initializer=initializer, weight_collections=weight_collections, trainable=trainable, - name='embedding_column_layer') - return embedding_column_layer(None, scope=scope) # pylint: disable=not-callable + name='embedding_column_layer', + ) + return embedding_column_layer(None, scope=scope) return _EmbeddingColumn( categorical_column=categorical_column, @@ -1331,7 +1370,8 @@ def _creator(weight_collections, scope): ckpt_to_load_from=ckpt_to_load_from, tensor_name_in_ckpt=tensor_name_in_ckpt, max_norm=max_norm, - trainable=trainable) + trainable=trainable, + ) def _numeric_column(key, @@ -1406,7 +1446,8 @@ def _numeric_column(key, shape=shape, default_value=default_value, dtype=dtype, - normalizer_fn=normalizer_fn) + normalizer_fn=normalizer_fn, + ) def _bucketized_column(source_column, boundaries): @@ -1482,8 +1523,8 @@ def _bucketized_column(source_column, boundaries): if len(source_column.shape) > 1: raise ValueError('source_column must be one-dimensional column. ' 'Given: {}'.format(source_column)) - if (not boundaries or - not (isinstance(boundaries, list) or isinstance(boundaries, tuple))): + if not boundaries or not (isinstance(boundaries, list) or + isinstance(boundaries, tuple)): raise ValueError('boundaries must be a sorted list.') for i in range(len(boundaries) - 1): if boundaries[i] >= boundaries[i + 1]: @@ -1549,12 +1590,14 @@ def _categorical_column_with_hash_bucket(key, return _HashedCategoricalColumn(key, hash_bucket_size, dtype) -def _categorical_column_with_vocabulary_file(key, - vocabulary_file, - vocabulary_size=None, - num_oov_buckets=0, - default_value=None, - dtype=dtypes.string): +def _categorical_column_with_vocabulary_file( + key, + vocabulary_file, + vocabulary_size=None, + num_oov_buckets=0, + default_value=None, + dtype=dtypes.string, +): """A `_CategoricalColumn` with a vocabulary file. Use this when your inputs are in string or integer format, and you have a @@ -1644,7 +1687,11 @@ def _categorical_column_with_vocabulary_file(key, vocabulary_size = sum(1 for _ in f) logging.info( 'vocabulary_size = %d in %s is inferred from the number of elements ' - 'in the vocabulary_file %s.', vocabulary_size, key, vocabulary_file) + 'in the vocabulary_file %s.', + vocabulary_size, + key, + vocabulary_file, + ) # `vocabulary_size` isn't required for lookup, but it is for `_num_buckets`. if vocabulary_size < 1: @@ -1652,7 +1699,7 @@ def _categorical_column_with_vocabulary_file(key, if num_oov_buckets: if default_value is not None: raise ValueError( - 'Can\'t specify both num_oov_buckets and default_value in {}.'.format( + "Can't specify both num_oov_buckets and default_value in {}.".format( key)) if num_oov_buckets < 0: raise ValueError('Invalid num_oov_buckets {} in {}.'.format( @@ -1665,7 +1712,8 @@ def _categorical_column_with_vocabulary_file(key, vocabulary_size=vocabulary_size, num_oov_buckets=0 if num_oov_buckets is None else num_oov_buckets, default_value=-1 if default_value is None else default_value, - dtype=dtype) + dtype=dtype, + ) def _categorical_column_with_vocabulary_list(key, @@ -1760,7 +1808,7 @@ def _categorical_column_with_vocabulary_list(key, if num_oov_buckets: if default_value != -1: raise ValueError( - 'Can\'t specify both num_oov_buckets and default_value in {}.'.format( + "Can't specify both num_oov_buckets and default_value in {}.".format( key)) if num_oov_buckets < 0: raise ValueError('Invalid num_oov_buckets {} in {}.'.format( @@ -1781,7 +1829,8 @@ def _categorical_column_with_vocabulary_list(key, vocabulary_list=tuple(vocabulary_list), dtype=dtype, default_value=default_value, - num_oov_buckets=num_oov_buckets) + num_oov_buckets=num_oov_buckets, + ) def _categorical_column_with_identity(key, num_buckets, default_value=None): @@ -1957,7 +2006,8 @@ def _weighted_categorical_column(categorical_column, return _WeightedCategoricalColumn( categorical_column=categorical_column, weight_feature_key=weight_feature_key, - dtype=dtype) + dtype=dtype, + ) def _crossed_column(keys, hash_bucket_size, hash_key=None): @@ -2071,8 +2121,8 @@ def _crossed_column(keys, hash_bucket_size, hash_key=None): raise ValueError( 'keys must be a list with length > 1. Given: {}'.format(keys)) for key in keys: - if (not isinstance(key, six.string_types) and - not isinstance(key, _CategoricalColumn)): + if not isinstance(key, six.string_types) and not isinstance( + key, _CategoricalColumn): raise ValueError( 'Unsupported key type. All keys must be either string, or ' 'categorical column except _HashedCategoricalColumn. ' @@ -2090,13 +2140,15 @@ def _crossed_column(keys, hash_bucket_size, hash_key=None): class _EmbeddingColumnLayer(base.Layer): """A layer that stores all the state required for a embedding column.""" - def __init__(self, - embedding_shape, - initializer, - weight_collections=None, - trainable=True, - name=None, - **kwargs): + def __init__( + self, + embedding_shape, + initializer, + weight_collections=None, + trainable=True, + name=None, + **kwargs, + ): """Constructor. Args: @@ -2132,7 +2184,8 @@ def build(self, _): shape=self._embedding_shape, dtype=dtypes.float32, initializer=self._initializer, - trainable=self.trainable) + trainable=self.trainable, + ) if self._weight_collections and not context.executing_eagerly(): _add_to_collections(self._embedding_weight_var, self._weight_collections) self.built = True @@ -2161,7 +2214,6 @@ class _FeatureColumn(object): @abc.abstractproperty def name(self): """Returns string, Used for naming and for name_scope.""" - pass @property def raw_name(self): @@ -2195,7 +2247,6 @@ def _transform_feature(self, inputs): Returns: Transformed feature `Tensor`. """ - pass @abc.abstractproperty def _parse_example_spec(self): @@ -2216,7 +2267,6 @@ def _parse_example_spec(self): return spec ``` """ - pass def _reset_config(self): """Resets the configuration in the column. @@ -2240,7 +2290,6 @@ class _DenseColumn(_FeatureColumn): @abc.abstractproperty def _variable_shape(self): """`TensorShape` of `_get_dense_tensor`, without batch dimension.""" - pass @abc.abstractmethod def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): @@ -2265,16 +2314,17 @@ def input_layer(features, feature_columns, ...): Returns: `Tensor` of shape [batch_size] + `_variable_shape`. """ - pass -def _create_weighted_sum(column, - builder, - units, - sparse_combiner, - weight_collections, - trainable, - weight_var=None): +def _create_weighted_sum( + column, + builder, + units, + sparse_combiner, + weight_collections, + trainable, + weight_var=None, +): """Creates a weighted sum for a dense/categorical column for linear_model.""" if isinstance(column, _CategoricalColumn): return _create_categorical_column_weighted_sum( @@ -2284,7 +2334,8 @@ def _create_weighted_sum(column, sparse_combiner=sparse_combiner, weight_collections=weight_collections, trainable=trainable, - weight_var=weight_var) + weight_var=weight_var, + ) else: return _create_dense_column_weighted_sum( column=column, @@ -2292,7 +2343,8 @@ def _create_weighted_sum(column, units=units, weight_collections=weight_collections, trainable=trainable, - weight_var=weight_var) + weight_var=weight_var, + ) def _create_dense_column_weighted_sum(column, @@ -2302,11 +2354,9 @@ def _create_dense_column_weighted_sum(column, trainable, weight_var=None): """Create a weighted sum of a dense column for linear_model.""" - tensor = column._get_dense_tensor( # pylint: disable=protected-access - builder, - weight_collections=weight_collections, - trainable=trainable) - num_elements = column._variable_shape.num_elements() # pylint: disable=protected-access + tensor = column._get_dense_tensor( + builder, weight_collections=weight_collections, trainable=trainable) + num_elements = column._variable_shape.num_elements() batch_size = array_ops.shape(tensor)[0] tensor = array_ops.reshape(tensor, shape=(batch_size, num_elements)) if weight_var is not None: @@ -2317,7 +2367,8 @@ def _create_dense_column_weighted_sum(column, shape=[num_elements, units], initializer=init_ops.zeros_initializer(), trainable=trainable, - collections=weight_collections) + collections=weight_collections, + ) return math_ops.matmul(tensor, weight, name='weighted_sum') @@ -2330,13 +2381,12 @@ class _CategoricalColumn(_FeatureColumn): A categorical feature typically handled with a `tf.SparseTensor` of IDs. """ - IdWeightPair = collections.namedtuple( # pylint: disable=invalid-name - 'IdWeightPair', ['id_tensor', 'weight_tensor']) + IdWeightPair = collections.namedtuple('IdWeightPair', + ['id_tensor', 'weight_tensor']) @abc.abstractproperty def _num_buckets(self): """Returns number of buckets in this sparse feature.""" - pass @abc.abstractmethod def _get_sparse_tensors(self, @@ -2363,17 +2413,17 @@ def _get_sparse_tensors(self, trainable: If `True` also add variables to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see `tf.compat.v1.get_variable`). """ - pass -def _create_categorical_column_weighted_sum(column, - builder, - units, - sparse_combiner, - weight_collections, - trainable, - weight_var=None): - # pylint: disable=g-doc-return-or-yield,g-doc-args +def _create_categorical_column_weighted_sum( + column, + builder, + units, + sparse_combiner, + weight_collections, + trainable, + weight_var=None, +): """Create a weighted sum of a categorical column for linear_model. Note to maintainer: As implementation details, the weighted sum is @@ -2400,10 +2450,8 @@ def _create_categorical_column_weighted_sum(column, For both cases, we can implement weighted sum via embedding_lookup with sparse_combiner = "sum". """ - sparse_tensors = column._get_sparse_tensors( # pylint: disable=protected-access - builder, - weight_collections=weight_collections, - trainable=trainable) + sparse_tensors = column._get_sparse_tensors( + builder, weight_collections=weight_collections, trainable=trainable) id_tensor = sparse_ops.sparse_reshape( sparse_tensors.id_tensor, [array_ops.shape(sparse_tensors.id_tensor)[0], -1]) @@ -2417,22 +2465,24 @@ def _create_categorical_column_weighted_sum(column, else: weight = variable_scope.get_variable( name='weights', - shape=(column._num_buckets, units), # pylint: disable=protected-access + shape=(column._num_buckets, units), initializer=init_ops.zeros_initializer(), trainable=trainable, - collections=weight_collections) + collections=weight_collections, + ) return embedding_ops.safe_embedding_lookup_sparse( weight, id_tensor, sparse_weights=weight_tensor, combiner=sparse_combiner, - name='weighted_sum') + name='weighted_sum', + ) class _SequenceDenseColumn(_FeatureColumn): """Represents dense sequence data.""" - TensorSequenceLengthPair = collections.namedtuple( # pylint: disable=invalid-name + TensorSequenceLengthPair = collections.namedtuple( 'TensorSequenceLengthPair', ['dense_tensor', 'sequence_length']) @abc.abstractmethod @@ -2441,7 +2491,6 @@ def _get_sequence_dense_tensor(self, weight_collections=None, trainable=None): """Returns a `TensorSequenceLengthPair`.""" - pass class _LazyBuilder(object): @@ -2523,7 +2572,7 @@ def get(self, key): column = key logging.debug('Transforming feature_column %s.', column) - transformed = column._transform_feature(self) # pylint: disable=protected-access + transformed = column._transform_feature(self) if transformed is None: raise ValueError('Column {} is not supported.'.format(column.name)) self._feature_tensors[column] = transformed @@ -2575,11 +2624,14 @@ def expand_dims(input_tensor): check_ops.assert_positive( array_ops.rank(feature_tensor), message='Feature (key: {}) cannot have rank 0. Given: {}'.format( - key, feature_tensor)) + key, feature_tensor), + ) ]): return control_flow_ops.cond( math_ops.equal(1, array_ops.rank(feature_tensor)), - lambda: expand_dims(feature_tensor), lambda: feature_tensor) + lambda: expand_dims(feature_tensor), + lambda: feature_tensor, + ) # TODO(ptucker): Move to third_party/tensorflow/python/ops/sparse_ops.py @@ -2617,10 +2669,14 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): input_tensor) if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): return input_tensor - with ops.name_scope(None, 'to_sparse_input', ( - input_tensor, - ignore_value, - )): + with ops.name_scope( + None, + 'to_sparse_input', + ( + input_tensor, + ignore_value, + ), + ): if ignore_value is None: if input_tensor.dtype == dtypes.string: # Exception due to TF strings are converted to numpy objects by default. @@ -2640,7 +2696,8 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): indices=indices, values=array_ops.gather_nd(input_tensor, indices, name='values'), dense_shape=array_ops.shape( - input_tensor, out_type=dtypes.int64, name='dense_shape')) + input_tensor, out_type=dtypes.int64, name='dense_shape'), + ) def _normalize_feature_columns(feature_columns): @@ -2692,8 +2749,9 @@ class _NumericColumn( _DenseColumn, collections.namedtuple( '_NumericColumn', - ['key', 'shape', 'default_value', 'dtype', 'normalizer_fn'])): - """see `numeric_column`.""" + ['key', 'shape', 'default_value', 'dtype', 'normalizer_fn']), +): + """See `numeric_column`.""" @property def name(self): @@ -2743,10 +2801,12 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): return inputs.get(self) -class _BucketizedColumn(_DenseColumn, _CategoricalColumn, - collections.namedtuple('_BucketizedColumn', - ['source_column', 'boundaries']) - ): +class _BucketizedColumn( + _DenseColumn, + _CategoricalColumn, + collections.namedtuple('_BucketizedColumn', + ['source_column', 'boundaries']), +): """See `bucketized_column`.""" @property @@ -2759,13 +2819,11 @@ def raw_name(self): @property def _parse_example_spec(self): - return self.source_column._parse_example_spec # pylint: disable=protected-access + return self.source_column._parse_example_spec def _transform_feature(self, inputs): source_tensor = inputs.get(self.source_column) - return math_ops._bucketize( # pylint: disable=protected-access - source_tensor, - boundaries=self.boundaries) + return math_ops._bucketize(source_tensor, boundaries=self.boundaries) @property def _variable_shape(self): @@ -2779,8 +2837,9 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): return array_ops.one_hot( indices=math_ops.cast(input_tensor, dtypes.int64), depth=len(self.boundaries) + 1, - on_value=1., - off_value=0.) + on_value=1.0, + off_value=0.0, + ) @property def _num_buckets(self): @@ -2800,13 +2859,15 @@ def _get_sparse_tensors(self, i1 = array_ops.reshape( array_ops.tile( array_ops.expand_dims(math_ops.range(0, batch_size), 1), - [1, source_dimension]), (-1,)) + [1, source_dimension], + ), + (-1,), + ) i2 = array_ops.tile(math_ops.range(0, source_dimension), [batch_size]) # Flatten the bucket indices and unique them across dimensions # E.g. 2nd dimension indices will range from k to 2*k-1 with k buckets - bucket_indices = ( - array_ops.reshape(input_tensor, - (-1,)) + (len(self.boundaries) + 1) * i2) + bucket_indices = array_ops.reshape(input_tensor, + (-1,)) + (len(self.boundaries) + 1) * i2 indices = math_ops.cast( array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64) @@ -2818,11 +2879,22 @@ def _get_sparse_tensors(self, class _EmbeddingColumn( - _DenseColumn, _SequenceDenseColumn, + _DenseColumn, + _SequenceDenseColumn, collections.namedtuple( '_EmbeddingColumn', - ('categorical_column', 'dimension', 'combiner', 'layer_creator', - 'ckpt_to_load_from', 'tensor_name_in_ckpt', 'max_norm', 'trainable'))): + ( + 'categorical_column', + 'dimension', + 'combiner', + 'layer_creator', + 'ckpt_to_load_from', + 'tensor_name_in_ckpt', + 'max_norm', + 'trainable', + ), + ), +): """See `embedding_column`.""" @property @@ -2833,7 +2905,7 @@ def name(self): @property def _parse_example_spec(self): - return self.categorical_column._parse_example_spec # pylint: disable=protected-access + return self.categorical_column._parse_example_spec def _transform_feature(self, inputs): return inputs.get(self.categorical_column) @@ -2850,21 +2922,20 @@ def _get_dense_tensor_internal(self, trainable=None): """Private method that follows the signature of _get_dense_tensor.""" # Get sparse IDs and weights. - sparse_tensors = self.categorical_column._get_sparse_tensors( # pylint: disable=protected-access - inputs, - weight_collections=weight_collections, - trainable=trainable) + sparse_tensors = self.categorical_column._get_sparse_tensors( + inputs, weight_collections=weight_collections, trainable=trainable) sparse_ids = sparse_tensors.id_tensor sparse_weights = sparse_tensors.weight_tensor embedding_weights = self.layer_creator( weight_collections=weight_collections, - scope=variable_scope.get_variable_scope()) + scope=variable_scope.get_variable_scope(), + ) if self.ckpt_to_load_from is not None: to_restore = embedding_weights if isinstance(to_restore, variables.PartitionedVariable): - to_restore = to_restore._get_variable_list() # pylint: disable=protected-access + to_restore = to_restore._get_variable_list() checkpoint_utils.init_from_checkpoint( self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) @@ -2875,7 +2946,8 @@ def _get_dense_tensor_internal(self, sparse_weights=sparse_weights, combiner=self.combiner, name='%s_weights' % self.name, - max_norm=self.max_norm) + max_norm=self.max_norm, + ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): if isinstance(self.categorical_column, _SequenceCategoricalColumn): @@ -2905,12 +2977,12 @@ def _get_sequence_dense_tensor(self, 'Suggested fix: Use one of sequence_categorical_column_with_*. ' 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column)) - dense_tensor = self._get_dense_tensor_internal( # pylint: disable=protected-access + dense_tensor = self._get_dense_tensor_internal( inputs=inputs, weight_collections=weight_collections, trainable=trainable) - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) sequence_length = fc_utils.sequence_length_from_sparse_tensor( sparse_tensors.id_tensor) return _SequenceDenseColumn.TensorSequenceLengthPair( @@ -2925,13 +2997,25 @@ def _get_graph_for_variable(var): class _SharedEmbeddingColumn( - _DenseColumn, _SequenceDenseColumn, + _DenseColumn, + _SequenceDenseColumn, collections.namedtuple( '_SharedEmbeddingColumn', - ('categorical_column', 'dimension', 'combiner', 'initializer', - 'shared_embedding_collection_name', 'ckpt_to_load_from', - 'tensor_name_in_ckpt', 'max_norm', 'trainable', 'partitioner', - 'ev_params'))): + ( + 'categorical_column', + 'dimension', + 'combiner', + 'initializer', + 'shared_embedding_collection_name', + 'ckpt_to_load_from', + 'tensor_name_in_ckpt', + 'max_norm', + 'trainable', + 'partitioner', + 'ev_params', + ), + ), +): """See `embedding_column`.""" @property @@ -2950,7 +3034,7 @@ def _var_scope_name(self): @property def _parse_example_spec(self): - return self.categorical_column._parse_example_spec # pylint: disable=protected-access + return self.categorical_column._parse_example_spec def _transform_feature(self, inputs): return inputs.get(self.categorical_column) @@ -2971,14 +3055,12 @@ def _get_dense_tensor_internal(self, # that the ops for different columns have distinct names. with ops.name_scope(None, default_name=self.name): # Get sparse IDs and weights. - sparse_tensors = self.categorical_column._get_sparse_tensors( # pylint: disable=protected-access - inputs, - weight_collections=weight_collections, - trainable=trainable) + sparse_tensors = self.categorical_column._get_sparse_tensors( + inputs, weight_collections=weight_collections, trainable=trainable) sparse_ids = sparse_tensors.id_tensor sparse_weights = sparse_tensors.weight_tensor - embedding_shape = (self.categorical_column._num_buckets, self.dimension) # pylint: disable=protected-access + embedding_shape = (self.categorical_column._num_buckets, self.dimension) shared_embedding_collection = ops.get_collection( self.shared_embedding_collection_name) if shared_embedding_collection: @@ -2991,16 +3073,19 @@ def _get_dense_tensor_internal(self, 'hood.'.format(shared_embedding_collection)) embedding_weights = shared_embedding_collection[0] if embedding_weights.get_shape( - ) != embedding_shape and not self.ev_params is not None: # noqa : E714 + ) != embedding_shape and self.ev_params is None: raise ValueError( 'Shared embedding collection {} contains variable {} of ' 'unexpected shape {}. Expected shape is {}. ' 'Suggested fix A: Choose a unique name for this collection. ' 'Suggested fix B: Do not add any variables to this collection. ' 'The feature_column library already adds a variable under the ' - 'hood.'.format(self.shared_embedding_collection_name, - embedding_weights.name, - embedding_weights.get_shape(), embedding_shape)) + 'hood.'.format( + self.shared_embedding_collection_name, + embedding_weights.name, + embedding_weights.get_shape(), + embedding_shape, + )) else: if self.ev_params is None: embedding_weights = variable_scope.get_variable( @@ -3010,13 +3095,14 @@ def _get_dense_tensor_internal(self, initializer=self.initializer, trainable=self.trainable and trainable, partitioner=self.partitioner, - collections=weight_collections) + collections=weight_collections, + ) else: # at eval or inference time, it is necessary to set # the initializers to zeros, so that new key will # get zero embedding - if os.environ.get('tf.estimator.mode', '') != \ - os.environ.get('tf.estimator.ModeKeys.TRAIN', 'train'): + if os.environ.get('tf.estimator.mode', '') != os.environ.get( + 'tf.estimator.ModeKeys.TRAIN', 'train'): initializer = init_ops.zeros_initializer() else: initializer = self.initializer @@ -3037,14 +3123,15 @@ def _get_dense_tensor_internal(self, partitioner=self.partitioner, collections=weight_collections, steps_to_live=self.ev_params.steps_to_live, - **extra_args) + **extra_args, + ) ops.add_to_collection(self.shared_embedding_collection_name, embedding_weights) if self.ckpt_to_load_from is not None: to_restore = embedding_weights if isinstance(to_restore, variables.PartitionedVariable): - to_restore = to_restore._get_variable_list() # pylint: disable=protected-access + to_restore = to_restore._get_variable_list() checkpoint_utils.init_from_checkpoint( self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) @@ -3056,7 +3143,8 @@ def _get_dense_tensor_internal(self, ragged_weights=sparse_weights, combiner=self.combiner, max_norm=self.max_norm, - name='%s_weights' % self.name) + name='%s_weights' % self.name, + ) # Return embedding lookup result. return embedding_ops.safe_embedding_lookup_sparse( @@ -3065,7 +3153,8 @@ def _get_dense_tensor_internal(self, sparse_weights=sparse_weights, combiner=self.combiner, name='%s_weights' % self.name, - max_norm=self.max_norm) + max_norm=self.max_norm, + ) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): if isinstance(self.categorical_column, _SequenceCategoricalColumn): @@ -3095,11 +3184,11 @@ def _get_sequence_dense_tensor(self, 'Suggested fix: Use one of sequence_categorical_column_with_*. ' 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column)) - dense_tensor = self._get_dense_tensor_internal( # pylint: disable=protected-access + dense_tensor = self._get_dense_tensor_internal( inputs=inputs, weight_collections=weight_collections, trainable=trainable) - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) sequence_length = fc_utils.sequence_length_from_sparse_tensor( sparse_tensors.id_tensor) return _SequenceDenseColumn.TensorSequenceLengthPair( @@ -3122,11 +3211,12 @@ def _check_shape(shape, key): return shape -class _HashedCategoricalColumn(_CategoricalColumn, - collections.namedtuple( - '_HashedCategoricalColumn', - ['key', 'hash_bucket_size', 'dtype'])): - """see `categorical_column_with_hash_bucket`.""" +class _HashedCategoricalColumn( + _CategoricalColumn, + collections.namedtuple('_HashedCategoricalColumn', + ['key', 'hash_bucket_size', 'dtype']), +): + """See `categorical_column_with_hash_bucket`.""" @property def name(self): @@ -3180,9 +3270,18 @@ def _get_sparse_tensors(self, class _VocabularyFileCategoricalColumn( _CategoricalColumn, - collections.namedtuple('_VocabularyFileCategoricalColumn', - ('key', 'vocabulary_file', 'vocabulary_size', - 'num_oov_buckets', 'dtype', 'default_value'))): + collections.namedtuple( + '_VocabularyFileCategoricalColumn', + ( + 'key', + 'vocabulary_file', + 'vocabulary_size', + 'num_oov_buckets', + 'dtype', + 'default_value', + ), + ), +): """See `categorical_column_with_vocabulary_file`.""" @property @@ -3218,7 +3317,8 @@ def _transform_feature(self, inputs): vocab_size=self.vocabulary_size, default_value=self.default_value, key_dtype=key_dtype, - name='{}_lookup'.format(self.key)).lookup(input_tensor) + name='{}_lookup'.format(self.key), + ).lookup(input_tensor) @property def _num_buckets(self): @@ -3236,7 +3336,8 @@ class _VocabularyListCategoricalColumn( _CategoricalColumn, collections.namedtuple( '_VocabularyListCategoricalColumn', - ('key', 'vocabulary_list', 'dtype', 'default_value', 'num_oov_buckets')) + ('key', 'vocabulary_list', 'dtype', 'default_value', 'num_oov_buckets'), + ), ): """See `categorical_column_with_vocabulary_list`.""" @@ -3272,7 +3373,8 @@ def _transform_feature(self, inputs): default_value=self.default_value, num_oov_buckets=self.num_oov_buckets, dtype=key_dtype, - name='{}_lookup'.format(self.key)).lookup(input_tensor) + name='{}_lookup'.format(self.key), + ).lookup(input_tensor) @property def _num_buckets(self): @@ -3286,10 +3388,11 @@ def _get_sparse_tensors(self, return _CategoricalColumn.IdWeightPair(inputs.get(self), None) -class _IdentityCategoricalColumn(_CategoricalColumn, - collections.namedtuple( - '_IdentityCategoricalColumn', - ('key', 'num_buckets', 'default_value'))): +class _IdentityCategoricalColumn( + _CategoricalColumn, + collections.namedtuple('_IdentityCategoricalColumn', + ('key', 'num_buckets', 'default_value')), +): """See `categorical_column_with_identity`.""" @property @@ -3317,7 +3420,8 @@ def _transform_feature(self, inputs): values, num_buckets, data=(values, num_buckets), - name='assert_less_than_num_buckets') + name='assert_less_than_num_buckets', + ) assert_greater = check_ops.assert_greater_equal( values, zero, data=(values,), name='assert_greater_or_equal_0') with ops.control_dependencies((assert_less, assert_greater)): @@ -3330,12 +3434,16 @@ def _transform_feature(self, inputs): array_ops.fill( dims=array_ops.shape(values), value=math_ops.cast(self.default_value, dtypes.int64), - name='default_values'), values) + name='default_values', + ), + values, + ) return sparse_tensor_lib.SparseTensor( indices=input_tensor.indices, values=values, - dense_shape=input_tensor.dense_shape) + dense_shape=input_tensor.dense_shape, + ) @property def _num_buckets(self): @@ -3353,7 +3461,9 @@ class _WeightedCategoricalColumn( _CategoricalColumn, collections.namedtuple( '_WeightedCategoricalColumn', - ('categorical_column', 'weight_feature_key', 'dtype'))): + ('categorical_column', 'weight_feature_key', 'dtype'), + ), +): """See `weighted_categorical_column`.""" @property @@ -3363,7 +3473,7 @@ def name(self): @property def _parse_example_spec(self): - config = self.categorical_column._parse_example_spec # pylint: disable=protected-access + config = self.categorical_column._parse_example_spec if self.weight_feature_key in config: raise ValueError('Parse config {} already exists for {}.'.format( config[self.weight_feature_key], self.weight_feature_key)) @@ -3372,7 +3482,7 @@ def _parse_example_spec(self): @property def _num_buckets(self): - return self.categorical_column._num_buckets # pylint: disable=protected-access + return self.categorical_column._num_buckets def _transform_feature(self, inputs): weight_tensor = inputs.get(self.weight_feature_key) @@ -3404,7 +3514,8 @@ def _get_sparse_tensors(self, class _CrossedColumn( _CategoricalColumn, collections.namedtuple('_CrossedColumn', - ['keys', 'hash_bucket_size', 'hash_key'])): + ['keys', 'hash_bucket_size', 'hash_key']), +): """See `crossed_column`.""" @property @@ -3422,7 +3533,7 @@ def _parse_example_spec(self): config = {} for key in self.keys: if isinstance(key, _FeatureColumn): - config.update(key._parse_example_spec) # pylint: disable=protected-access + config.update(key._parse_example_spec) else: # key must be a string config.update({key: parsing_ops.VarLenFeature(dtypes.string)}) return config @@ -3433,7 +3544,7 @@ def _transform_feature(self, inputs): if isinstance(key, six.string_types): feature_tensors.append(inputs.get(key)) elif isinstance(key, _CategoricalColumn): - ids_and_weights = key._get_sparse_tensors(inputs) # pylint: disable=protected-access + ids_and_weights = key._get_sparse_tensors(inputs) if ids_and_weights.weight_tensor is not None: raise ValueError( 'crossed_column does not support weight_tensor, but the given ' @@ -3445,7 +3556,8 @@ def _transform_feature(self, inputs): return sparse_ops.sparse_cross_hashed( inputs=feature_tensors, num_buckets=self.hash_bucket_size, - hash_key=self.hash_key) + hash_key=self.hash_key, + ) @property def _num_buckets(self): @@ -3477,9 +3589,11 @@ def _collect_leaf_level_keys(cross): return leaf_level_keys -class _IndicatorColumn(_DenseColumn, _SequenceDenseColumn, - collections.namedtuple('_IndicatorColumn', - ['categorical_column'])): +class _IndicatorColumn( + _DenseColumn, + _SequenceDenseColumn, + collections.namedtuple('_IndicatorColumn', ['categorical_column']), +): """Represents a one-hot column for use in deep networks. Args: @@ -3503,7 +3617,7 @@ def _transform_feature(self, inputs): Raises: ValueError: if input rank is not known at graph building time. """ - id_weight_pair = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access + id_weight_pair = self.categorical_column._get_sparse_tensors(inputs) id_tensor = id_weight_pair.id_tensor weight_tensor = id_weight_pair.weight_tensor @@ -3512,15 +3626,18 @@ def _transform_feature(self, inputs): weighted_column = sparse_ops.sparse_merge( sp_ids=id_tensor, sp_values=weight_tensor, - vocab_size=int(self._variable_shape[-1])) + vocab_size=int(self._variable_shape[-1]), + ) # Remove (?, -1) index. weighted_column = sparse_ops.sparse_slice(weighted_column, [0, 0], weighted_column.dense_shape) # Use scatter_nd to merge duplicated indices if existed, # instead of sparse_tensor_to_dense. - return array_ops.scatter_nd(weighted_column.indices, - weighted_column.values, - weighted_column.dense_shape) + return array_ops.scatter_nd( + weighted_column.indices, + weighted_column.values, + weighted_column.dense_shape, + ) dense_id_tensor = sparse_ops.sparse_tensor_to_dense( id_tensor, default_value=-1) @@ -3538,12 +3655,12 @@ def _transform_feature(self, inputs): @property def _parse_example_spec(self): - return self.categorical_column._parse_example_spec # pylint: disable=protected-access + return self.categorical_column._parse_example_spec @property def _variable_shape(self): """Returns a `TensorShape` representing the shape of the dense `Tensor`.""" - return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) # pylint: disable=protected-access + return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): """Returns dense `Tensor` representing feature. @@ -3598,7 +3715,7 @@ def _get_sequence_dense_tensor(self, # Feature has been already transformed. Return the intermediate # representation created by _transform_feature. dense_tensor = inputs.get(self) - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) sequence_length = fc_utils.sequence_length_from_sparse_tensor( sparse_tensors.id_tensor) return _SequenceDenseColumn.TensorSequenceLengthPair( @@ -3627,14 +3744,18 @@ def _verify_static_batch_size_equality(tensors, columns): raise ValueError( 'Batch size (first dimension) of each feature must be same. ' 'Batch size of columns ({}, {}): ({}, {})'.format( - columns[bath_size_column_index].name, columns[i].name, - expected_batch_size, tensors[i].shape.dims[0])) + columns[bath_size_column_index].name, + columns[i].name, + expected_batch_size, + tensors[i].shape.dims[0], + )) -class _SequenceCategoricalColumn(_CategoricalColumn, - collections.namedtuple( - '_SequenceCategoricalColumn', - ['categorical_column'])): +class _SequenceCategoricalColumn( + _CategoricalColumn, + collections.namedtuple('_SequenceCategoricalColumn', + ['categorical_column']), +): """Represents sequences of categorical data.""" @property @@ -3643,20 +3764,20 @@ def name(self): @property def _parse_example_spec(self): - return self.categorical_column._parse_example_spec # pylint: disable=protected-access + return self.categorical_column._parse_example_spec def _transform_feature(self, inputs): - return self.categorical_column._transform_feature(inputs) # pylint: disable=protected-access + return self.categorical_column._transform_feature(inputs) @property def _num_buckets(self): - return self.categorical_column._num_buckets # pylint: disable=protected-access + return self.categorical_column._num_buckets def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) id_tensor = sparse_tensors.id_tensor weight_tensor = sparse_tensors.weight_tensor diff --git a/easy_rec/python/compat/feature_column/feature_column_v2.py b/easy_rec/python/compat/feature_column/feature_column_v2.py index 01b9fc93f..21d4964b1 100644 --- a/easy_rec/python/compat/feature_column/feature_column_v2.py +++ b/easy_rec/python/compat/feature_column/feature_column_v2.py @@ -147,19 +147,6 @@ from tensorflow.python.keras import utils from tensorflow.python.keras.engine import training from tensorflow.python.keras.engine.base_layer import Layer -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import embedding_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import lookup_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import sparse_ops -from tensorflow.python.ops import string_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables from tensorflow.python.platform import gfile from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training import checkpoint_utils @@ -172,7 +159,14 @@ from easy_rec.python.compat.feature_column import utils as fc_utils from easy_rec.python.layers import utils as layer_utils -from easy_rec.python.compat.feature_column.feature_column import embedding_lookup_ragged # NOQA +from tensorflow.python.ops import ( # NOQA + array_ops, check_ops, control_flow_ops, embedding_ops, init_ops, lookup_ops, + math_ops, nn_ops, parsing_ops, sparse_ops, string_ops, variable_scope, + variables, +) + +from easy_rec.python.compat.feature_column.feature_column import ( # NOQA + embedding_lookup_ragged,) _FEATURE_COLUMN_DEPRECATION_DATE = None _FEATURE_COLUMN_DEPRECATION = ('The old _FeatureColumn APIs are being ' @@ -194,14 +188,16 @@ class StateManager(object): only. """ - def create_variable(self, - feature_column, - name, - shape, - dtype=None, - trainable=True, - use_resource=True, - initializer=None): + def create_variable( + self, + feature_column, + name, + shape, + dtype=None, + trainable=True, + use_resource=True, + initializer=None, + ): """Creates a new variable. Args: @@ -283,14 +279,16 @@ def __init__(self, layer, trainable): self._layer = layer self._cols_to_vars_map = collections.defaultdict(lambda: {}) - def create_variable(self, - feature_column, - name, - shape, - dtype=None, - trainable=True, - use_resource=True, - initializer=None): + def create_variable( + self, + feature_column, + name, + shape, + dtype=None, + trainable=True, + use_resource=True, + initializer=None, + ): if name in self._cols_to_vars_map[feature_column]: raise ValueError('Variable already exists.') @@ -304,7 +302,8 @@ def create_variable(self, # TODO(rohanj): Get rid of this hack once we have a mechanism for # specifying a default partitioner for an entire layer. In that case, # the default getter for Layers should work. - getter=variable_scope.get_variable) + getter=variable_scope.get_variable, + ) self._cols_to_vars_map[feature_column][name] = var return var @@ -349,8 +348,8 @@ def __init__(self, feature_columns, expected_column_type, trainable, name, def build(self, _): for column in self._feature_columns: - with variable_scope._pure_variable_scope(self.name): # pylint: disable=protected-access - with variable_scope._pure_variable_scope(column.name): # pylint: disable=protected-access + with variable_scope._pure_variable_scope(self.name): + with variable_scope._pure_variable_scope(column.name): column.create_state(self._state_manager) super(_BaseFeaturesLayer, self).build(None) @@ -440,7 +439,8 @@ def __init__(self, feature_columns, trainable=True, name=None, **kwargs): trainable=trainable, name=name, expected_column_type=DenseColumn, - **kwargs) + **kwargs, + ) @property def _is_feature_layer(self): @@ -487,13 +487,15 @@ def call(self, features, cols_to_output_tensors=None): class _LinearModelLayer(Layer): """Layer that contains logic for `LinearModel`.""" - def __init__(self, - feature_columns, - units=1, - sparse_combiner='sum', - trainable=True, - name=None, - **kwargs): + def __init__( + self, + feature_columns, + units=1, + sparse_combiner='sum', + trainable=True, + name=None, + **kwargs, + ): super(_LinearModelLayer, self).__init__( name=name, trainable=trainable, **kwargs) @@ -515,9 +517,9 @@ def build(self, _): # information to percolate down. We also use _pure_variable_scope's here # since we want to open up a name_scope in the `call` method while creating # the ops. - with variable_scope._pure_variable_scope(self.name): # pylint: disable=protected-access + with variable_scope._pure_variable_scope(self.name): for column in self._feature_columns: - with variable_scope._pure_variable_scope(column.name): # pylint: disable=protected-access + with variable_scope._pure_variable_scope(column.name): # Create the state for each feature column column.create_state(self._state_manager) @@ -532,7 +534,8 @@ def build(self, _): dtype=dtypes.float32, shape=(first_dim, self._units), initializer=init_ops.zeros_initializer(), - trainable=self.trainable) + trainable=self.trainable, + ) # Create a bias variable. self.bias = self.add_variable( @@ -545,7 +548,8 @@ def build(self, _): # TODO(rohanj): Get rid of this hack once we have a mechanism for # specifying a default partitioner for an entire layer. In that case, # the default getter for Layers should work. - getter=variable_scope.get_variable) + getter=variable_scope.get_variable, + ) super(_LinearModelLayer, self).build(None) @@ -567,7 +571,8 @@ def call(self, features): transformation_cache=transformation_cache, state_manager=self._state_manager, sparse_combiner=self._sparse_combiner, - weight_var=weight_var) + weight_var=weight_var, + ) weighted_sums.append(weighted_sum) _verify_static_batch_size_equality(weighted_sums, self._feature_columns) @@ -617,13 +622,15 @@ class LinearModel(training.Model): ``` """ - def __init__(self, - feature_columns, - units=1, - sparse_combiner='sum', - trainable=True, - name=None, - **kwargs): + def __init__( + self, + feature_columns, + units=1, + sparse_combiner='sum', + trainable=True, + name=None, + **kwargs, + ): """Constructs a LinearLayer. Args: @@ -816,16 +823,18 @@ def make_parse_example_spec_v2(feature_columns): return result -def embedding_column(categorical_column, - dimension, - combiner='mean', - initializer=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - max_norm=None, - trainable=True, - partitioner=None, - ev_params=None): +def embedding_column( + categorical_column, + dimension, + combiner='mean', + initializer=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True, + partitioner=None, + ev_params=None, +): """`DenseColumn` that converts from sparse, categorical input. Use this when your inputs are sparse, but you want to convert them to a dense @@ -921,20 +930,23 @@ def model_fn(features, ...): max_norm=max_norm, trainable=trainable, partitioner=partitioner, - ev_params=ev_params) - - -def shared_embedding_columns(categorical_columns, - dimension, - combiner='mean', - initializer=None, - shared_embedding_collection_name=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - max_norm=None, - trainable=True, - partitioner=None, - ev_params=None): + ev_params=ev_params, + ) + + +def shared_embedding_columns( + categorical_columns, + dimension, + combiner='mean', + initializer=None, + shared_embedding_collection_name=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True, + partitioner=None, + ev_params=None, +): """List of dense columns that convert from sparse, categorical input. This is similar to `embedding_column`, except that it produces a list of @@ -1052,24 +1064,24 @@ def model_fn(features, ...): sorted_columns = sorted(categorical_columns, key=lambda x: x.name) c0 = sorted_columns[0] - num_buckets = c0._num_buckets # pylint: disable=protected-access - if not isinstance(c0, fc_old._CategoricalColumn): # pylint: disable=protected-access + num_buckets = c0._num_buckets + if not isinstance(c0, fc_old._CategoricalColumn): raise ValueError( 'All categorical_columns must be subclasses of _CategoricalColumn. ' 'Given: {}, of type: {}'.format(c0, type(c0))) if isinstance(c0, - (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)): # pylint: disable=protected-access + (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)): c0 = c0.categorical_column for c in sorted_columns[1:]: if isinstance( - c, (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)): # pylint: disable=protected-access + c, (fc_old._WeightedCategoricalColumn, WeightedCategoricalColumn)): c = c.categorical_column - if num_buckets != c._num_buckets: # pylint: disable=protected-access + if num_buckets != c._num_buckets: raise ValueError( 'To use shared_embedding_column, all categorical_columns must have ' 'the same number of buckets. Given column: {} with buckets: {} does ' 'not match column: {} with buckets: {}'.format( - c0, num_buckets, c, c._num_buckets)) # pylint: disable=protected-access + c0, num_buckets, c, c._num_buckets)) if not shared_embedding_collection_name: shared_embedding_collection_name = '_'.join(c.name for c in sorted_columns) @@ -1078,7 +1090,7 @@ def model_fn(features, ...): result = [] for column in categorical_columns: result.append( - fc_old._SharedEmbeddingColumn( # pylint: disable=protected-access + fc_old._SharedEmbeddingColumn( categorical_column=column, initializer=initializer, dimension=dimension, @@ -1089,20 +1101,23 @@ def model_fn(features, ...): max_norm=max_norm, trainable=trainable, partitioner=partitioner, - ev_params=ev_params)) + ev_params=ev_params, + )) return result -def shared_embedding_columns_v2(categorical_columns, - dimension, - combiner='mean', - initializer=None, - shared_embedding_collection_name=None, - ckpt_to_load_from=None, - tensor_name_in_ckpt=None, - max_norm=None, - trainable=True): +def shared_embedding_columns_v2( + categorical_columns, + dimension, + combiner='mean', + initializer=None, + shared_embedding_collection_name=None, + ckpt_to_load_from=None, + tensor_name_in_ckpt=None, + max_norm=None, + trainable=True, +): """List of dense columns that convert from sparse, categorical input. This is similar to `embedding_column`, except that it produces a list of @@ -1247,8 +1262,14 @@ def model_fn(features, ...): shared_embedding_collection_name += '_shared_embedding' column_creator = SharedEmbeddingColumnCreator( - dimension, initializer, ckpt_to_load_from, tensor_name_in_ckpt, - num_buckets, trainable, shared_embedding_collection_name) + dimension, + initializer, + ckpt_to_load_from, + tensor_name_in_ckpt, + num_buckets, + trainable, + shared_embedding_collection_name, + ) result = [] for column in categorical_columns: @@ -1259,12 +1280,14 @@ def model_fn(features, ...): return result -def numeric_column(key, - shape=(1,), - default_value=None, - dtype=dtypes.float32, - normalizer_fn=None, - feature_name=None): +def numeric_column( + key, + shape=(1,), + default_value=None, + dtype=dtypes.float32, + normalizer_fn=None, + feature_name=None, +): """Represents real valued or numerical features. Example: @@ -1333,7 +1356,8 @@ def numeric_column(key, shape=shape, default_value=default_value, dtype=dtype, - normalizer_fn=normalizer_fn) + normalizer_fn=normalizer_fn, + ) def bucketized_column(source_column, boundaries): @@ -1402,7 +1426,7 @@ def bucketized_column(source_column, boundaries): one-dimensional. ValueError: If `boundaries` is not a sorted list or tuple. """ - if not isinstance(source_column, (NumericColumn, fc_old._NumericColumn)): # pylint: disable=protected-access + if not isinstance(source_column, (NumericColumn, fc_old._NumericColumn)): raise ValueError( 'source_column must be a column generated with numeric_column(). ' 'Given: {}'.format(source_column)) @@ -1478,13 +1502,15 @@ def categorical_column_with_hash_bucket(key, return HashedCategoricalColumn(feature_name, key, hash_bucket_size, dtype) -def categorical_column_with_vocabulary_file_v2(key, - vocabulary_file, - vocabulary_size=None, - dtype=dtypes.string, - default_value=None, - num_oov_buckets=0, - feature_name=None): +def categorical_column_with_vocabulary_file_v2( + key, + vocabulary_file, + vocabulary_size=None, + dtype=dtypes.string, + default_value=None, + num_oov_buckets=0, + feature_name=None, +): """A `CategoricalColumn` with a vocabulary file. Use this when your inputs are in string or integer format, and you have a @@ -1574,7 +1600,11 @@ def categorical_column_with_vocabulary_file_v2(key, vocabulary_size = sum(1 for _ in f) logging.info( 'vocabulary_size = %d in %s is inferred from the number of elements ' - 'in the vocabulary_file %s.', vocabulary_size, key, vocabulary_file) + 'in the vocabulary_file %s.', + vocabulary_size, + key, + vocabulary_file, + ) # `vocabulary_size` isn't required for lookup, but it is for `_num_buckets`. if vocabulary_size < 1: @@ -1582,7 +1612,7 @@ def categorical_column_with_vocabulary_file_v2(key, if num_oov_buckets: if default_value is not None: raise ValueError( - 'Can\'t specify both num_oov_buckets and default_value in {}.'.format( + "Can't specify both num_oov_buckets and default_value in {}.".format( key)) if num_oov_buckets < 0: raise ValueError('Invalid num_oov_buckets {} in {}.'.format( @@ -1596,15 +1626,18 @@ def categorical_column_with_vocabulary_file_v2(key, vocabulary_size=vocabulary_size, num_oov_buckets=0 if num_oov_buckets is None else num_oov_buckets, default_value=-1 if default_value is None else default_value, - dtype=dtype) + dtype=dtype, + ) -def categorical_column_with_vocabulary_list(key, - vocabulary_list, - dtype=None, - default_value=-1, - num_oov_buckets=0, - feature_name=None): +def categorical_column_with_vocabulary_list( + key, + vocabulary_list, + dtype=None, + default_value=-1, + num_oov_buckets=0, + feature_name=None, +): """A `CategoricalColumn` with in-memory vocabulary. Use this when your inputs are in string or integer format, and you have an @@ -1692,7 +1725,7 @@ def categorical_column_with_vocabulary_list(key, if num_oov_buckets: if default_value != -1: raise ValueError( - 'Can\'t specify both num_oov_buckets and default_value in {}.'.format( + "Can't specify both num_oov_buckets and default_value in {}.".format( key)) if num_oov_buckets < 0: raise ValueError('Invalid num_oov_buckets {} in {}.'.format( @@ -1714,7 +1747,8 @@ def categorical_column_with_vocabulary_list(key, vocabulary_list=tuple(vocabulary_list), dtype=dtype, default_value=default_value, - num_oov_buckets=num_oov_buckets) + num_oov_buckets=num_oov_buckets, + ) def categorical_column_with_identity(key, @@ -1787,7 +1821,8 @@ def categorical_column_with_identity(key, feature_name=feature_name, key=key, number_buckets=num_buckets, - default_value=default_value) + default_value=default_value, + ) def indicator_column(categorical_column): @@ -1896,7 +1931,8 @@ def weighted_categorical_column(categorical_column, return WeightedCategoricalColumn( categorical_column=categorical_column, weight_feature_key=weight_feature_key, - dtype=dtype) + dtype=dtype, + ) def crossed_column(keys, hash_bucket_size, hash_key=None, feature_name=None): @@ -2010,14 +2046,14 @@ def crossed_column(keys, hash_bucket_size, hash_key=None, feature_name=None): raise ValueError( 'keys must be a list with length > 1. Given: {}'.format(keys)) for key in keys: - if (not isinstance(key, six.string_types) and - not isinstance(key, (CategoricalColumn, fc_old._CategoricalColumn))): # pylint: disable=protected-access + if not isinstance(key, six.string_types) and not isinstance( + key, (CategoricalColumn, fc_old._CategoricalColumn)): raise ValueError( 'Unsupported key type. All keys must be either string, or ' 'categorical column except HashedCategoricalColumn. ' 'Given: {}'.format(key)) if isinstance(key, - (HashedCategoricalColumn, fc_old._HashedCategoricalColumn)): # pylint: disable=protected-access + (HashedCategoricalColumn, fc_old._HashedCategoricalColumn)): raise ValueError( 'categorical_column_with_hash_bucket is not supported for crossing. ' 'Hashing before crossing will increase probability of collision. ' @@ -2026,7 +2062,8 @@ def crossed_column(keys, hash_bucket_size, hash_key=None, feature_name=None): feature_name=feature_name, keys=tuple(keys), hash_bucket_size=hash_bucket_size, - hash_key=hash_key) + hash_key=hash_key, + ) @six.add_metaclass(abc.ABCMeta) @@ -2049,7 +2086,6 @@ class FeatureColumn(object): @abc.abstractproperty def name(self): """Returns string, Used for naming.""" - pass @property def raw_name(self): @@ -2081,7 +2117,6 @@ def transform_feature(self, transformation_cache, state_manager): Returns: Transformed feature `Tensor`. """ - pass @abc.abstractproperty def parse_example_spec(self): @@ -2102,7 +2137,6 @@ def parse_example_spec(self): return spec ``` """ - pass def create_state(self, state_manager): """Uses the `state_manager` to create state for the FeatureColumn. @@ -2111,7 +2145,6 @@ def create_state(self, state_manager): state_manager: A `StateManager` to create / access resources such as lookup tables and variables. """ - pass @abc.abstractproperty def _is_v2_column(self): @@ -2121,7 +2154,6 @@ def _is_v2_column(self): might take in old categorical columns as input and then we want to use the old API. """ - pass @abc.abstractproperty def parents(self): @@ -2135,7 +2167,6 @@ def parents(self): a.parents = ['f1'] c.parents = [a, 'f2'] """ - pass @abc.abstractmethod def _get_config(self): @@ -2192,7 +2223,6 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): A serializable Dict that can be used to deserialize the object with from_config. """ - pass @classmethod def _from_config(cls, config, custom_objects=None, columns_by_name=None): @@ -2216,7 +2246,6 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): Returns: A FeatureColumn for the input config. """ - pass class DenseColumn(FeatureColumn): @@ -2229,7 +2258,6 @@ class DenseColumn(FeatureColumn): @abc.abstractproperty def variable_shape(self): """`TensorShape` of `get_dense_tensor`, without batch dimension.""" - pass @abc.abstractmethod def get_dense_tensor(self, transformation_cache, state_manager): @@ -2253,7 +2281,6 @@ def input_layer(features, feature_columns, ...): Returns: `Tensor` of shape [batch_size] + `variable_shape`. """ - pass def is_feature_column_v2(feature_columns): @@ -2261,7 +2288,7 @@ def is_feature_column_v2(feature_columns): for feature_column in feature_columns: if not isinstance(feature_column, FeatureColumn): return False - if not feature_column._is_v2_column: # pylint: disable=protected-access + if not feature_column._is_v2_column: return False return True @@ -2275,13 +2302,15 @@ def _create_weighted_sum(column, transformation_cache, state_manager, transformation_cache=transformation_cache, state_manager=state_manager, sparse_combiner=sparse_combiner, - weight_var=weight_var) + weight_var=weight_var, + ) else: return _create_dense_column_weighted_sum( column=column, transformation_cache=transformation_cache, state_manager=state_manager, - weight_var=weight_var) + weight_var=weight_var, + ) def _create_dense_column_weighted_sum(column, transformation_cache, @@ -2300,13 +2329,12 @@ class CategoricalColumn(FeatureColumn): A categorical feature typically handled with a `tf.SparseTensor` of IDs. """ - IdWeightPair = collections.namedtuple( # pylint: disable=invalid-name - 'IdWeightPair', ('id_tensor', 'weight_tensor')) + IdWeightPair = collections.namedtuple('IdWeightPair', + ('id_tensor', 'weight_tensor')) @abc.abstractproperty def num_buckets(self): """Returns number of buckets in this sparse feature.""" - pass @abc.abstractmethod def get_sparse_tensors(self, transformation_cache, state_manager): @@ -2328,13 +2356,11 @@ def get_sparse_tensors(self, transformation_cache, state_manager): state_manager: A `StateManager` to create / access resources such as lookup tables. """ - pass def _create_categorical_column_weighted_sum(column, transformation_cache, state_manager, sparse_combiner, weight_var): - # pylint: disable=g-doc-return-or-yield,g-doc-args """Create a weighted sum of a categorical column for linear_model. Note to maintainer: As implementation details, the weighted sum is @@ -2376,13 +2402,14 @@ def _create_categorical_column_weighted_sum(column, transformation_cache, id_tensor, sparse_weights=weight_tensor, combiner=sparse_combiner, - name='weighted_sum') + name='weighted_sum', + ) class SequenceDenseColumn(FeatureColumn): """Represents dense sequence data.""" - TensorSequenceLengthPair = collections.namedtuple( # pylint: disable=invalid-name + TensorSequenceLengthPair = collections.namedtuple( 'TensorSequenceLengthPair', ('dense_tensor', 'sequence_length')) @abc.abstractmethod @@ -2395,7 +2422,6 @@ def get_sequence_dense_tensor(self, transformation_cache, state_manager): state_manager: A `StateManager` to create / access resources such as lookup tables. """ - pass class FeatureTransformationCache(object): @@ -2527,11 +2553,14 @@ def expand_dims(input_tensor): check_ops.assert_positive( array_ops.rank(feature_tensor), message='Feature (key: {}) cannot have rank 0. Given: {}'.format( - key, feature_tensor)) + key, feature_tensor), + ) ]): return control_flow_ops.cond( math_ops.equal(1, array_ops.rank(feature_tensor)), - lambda: expand_dims(feature_tensor), lambda: feature_tensor) + lambda: expand_dims(feature_tensor), + lambda: feature_tensor, + ) # TODO(ptucker): Move to third_party/tensorflow/python/ops/sparse_ops.py @@ -2559,10 +2588,14 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): input_tensor) if isinstance(input_tensor, sparse_tensor_lib.SparseTensor): return input_tensor - with ops.name_scope(None, 'to_sparse_input', ( - input_tensor, - ignore_value, - )): + with ops.name_scope( + None, + 'to_sparse_input', + ( + input_tensor, + ignore_value, + ), + ): if ignore_value is None: if input_tensor.dtype == dtypes.string: # Exception due to TF strings are converted to numpy objects by default. @@ -2582,7 +2615,8 @@ def _to_sparse_input_and_drop_ignore_values(input_tensor, ignore_value=None): indices=indices, values=array_ops.gather_nd(input_tensor, indices, name='values'), dense_shape=array_ops.shape( - input_tensor, out_type=dtypes.int64, name='dense_shape')) + input_tensor, out_type=dtypes.int64, name='dense_shape'), + ) def _normalize_feature_columns(feature_columns): @@ -2632,11 +2666,14 @@ def _normalize_feature_columns(feature_columns): class NumericColumn( DenseColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - collections.namedtuple('NumericColumn', - ('feature_name', 'key', 'shape', 'default_value', - 'dtype', 'normalizer_fn'))): - """see `numeric_column`.""" + fc_old._DenseColumn, + collections.namedtuple( + 'NumericColumn', + ('feature_name', 'key', 'shape', 'default_value', 'dtype', + 'normalizer_fn'), + ), +): + """See `numeric_column`.""" @property def _is_v2_column(self): @@ -2762,16 +2799,16 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class BucketizedColumn( DenseColumn, CategoricalColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple('BucketizedColumn', - ('source_column', 'boundaries'))): + fc_old._DenseColumn, + fc_old._CategoricalColumn, + collections.namedtuple('BucketizedColumn', ('source_column', 'boundaries')), +): """See `bucketized_column`.""" @property def _is_v2_column(self): - return (isinstance(self.source_column, FeatureColumn) and - self.source_column._is_v2_column) # pylint: disable=protected-access + return isinstance(self.source_column, + FeatureColumn) and self.source_column._is_v2_column @property def name(self): @@ -2792,23 +2829,19 @@ def parse_example_spec(self): @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - return self.source_column._parse_example_spec # pylint: disable=protected-access + return self.source_column._parse_example_spec @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): """Returns bucketized categorical `source_column` tensor.""" source_tensor = inputs.get(self.source_column) - return math_ops._bucketize( # pylint: disable=protected-access - source_tensor, - boundaries=self.boundaries) + return math_ops._bucketize(source_tensor, boundaries=self.boundaries) def transform_feature(self, transformation_cache, state_manager): """Returns bucketized categorical `source_column` tensor.""" source_tensor = transformation_cache.get(self.source_column, state_manager) - return math_ops._bucketize( # pylint: disable=protected-access - source_tensor, - boundaries=self.boundaries) + return math_ops._bucketize(source_tensor, boundaries=self.boundaries) @property def variable_shape(self): @@ -2826,8 +2859,9 @@ def _get_dense_tensor_for_input_tensor(self, input_tensor): return array_ops.one_hot( indices=math_ops.cast(input_tensor, dtypes.int64), depth=len(self.boundaries) + 1, - on_value=1., - off_value=0.) + on_value=1.0, + off_value=0.0, + ) def get_dense_tensor(self, transformation_cache, state_manager): """Returns one hot encoded dense `Tensor`.""" @@ -2862,13 +2896,15 @@ def _get_sparse_tensors_for_input_tensor(self, input_tensor): i1 = array_ops.reshape( array_ops.tile( array_ops.expand_dims(math_ops.range(0, batch_size), 1), - [1, source_dimension]), (-1,)) + [1, source_dimension], + ), + (-1,), + ) i2 = array_ops.tile(math_ops.range(0, source_dimension), [batch_size]) # Flatten the bucket indices and unique them across dimensions # E.g. 2nd dimension indices will range from k to 2*k-1 with k buckets - bucket_indices = ( - array_ops.reshape(input_tensor, - (-1,)) + (len(self.boundaries) + 1) * i2) + bucket_indices = array_ops.reshape(input_tensor, + (-1,)) + (len(self.boundaries) + 1) * i2 indices = math_ops.cast( array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64) @@ -2919,16 +2955,17 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class SequenceBucketizedColumn( DenseColumn, CategoricalColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._CategoricalColumn, # pylint: disable=protected-access + fc_old._DenseColumn, + fc_old._CategoricalColumn, collections.namedtuple('SequenceBucketizedColumn', - ('source_column', 'boundaries'))): + ('source_column', 'boundaries')), +): """See `bucketized_column`.""" @property def _is_v2_column(self): - return (isinstance(self.source_column, FeatureColumn) and - self.source_column._is_v2_column) # pylint: disable=protected-access + return isinstance(self.source_column, + FeatureColumn) and self.source_column._is_v2_column @property def name(self): @@ -2949,7 +2986,7 @@ def parse_example_spec(self): @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - return self.source_column._parse_example_spec # pylint: disable=protected-access + return self.source_column._parse_example_spec @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) @@ -2961,15 +2998,14 @@ def _transform_feature(self, inputs): bucketize_tensor = sparse_tensor_lib.SparseTensor( indices=source_tensor.indices, values=bucketize_values, - dense_shape=source_tensor.dense_shape) + dense_shape=source_tensor.dense_shape, + ) return bucketize_tensor def transform_feature(self, transformation_cache, state_manager): """Returns bucketized categorical `source_column` tensor.""" source_tensor = transformation_cache.get(self.source_column, state_manager) - return math_ops._bucketize( # pylint: disable=protected-access - source_tensor, - boundaries=self.boundaries) + return math_ops._bucketize(source_tensor, boundaries=self.boundaries) @property def variable_shape(self): @@ -2987,8 +3023,9 @@ def _get_dense_tensor_for_input_tensor(self, input_tensor): return array_ops.one_hot( indices=math_ops.cast(input_tensor, dtypes.int64), depth=len(self.boundaries) + 1, - on_value=1., - off_value=0.) + on_value=1.0, + off_value=0.0, + ) def get_dense_tensor(self, transformation_cache, state_manager): """Returns one hot encoded dense `Tensor`.""" @@ -3025,14 +3062,14 @@ def _get_sparse_tensors_for_input_tensor(self, input_sparse_tensor): i2 = array_ops.tile(math_ops.range(0, source_dimension), [batch_size]) # Flatten the bucket indices and unique them across dimensions # E.g. 2nd dimension indices will range from k to 2*k-1 with k buckets - bucket_indices = ( - array_ops.reshape(input_tensor, - (-1,)) + (len(self.boundaries) + 1) * i2) + bucket_indices = array_ops.reshape(input_tensor, + (-1,)) + (len(self.boundaries) + 1) * i2 sparse_tensor = sparse_tensor_lib.SparseTensor( indices=input_indices, values=bucket_indices, - dense_shape=input_sparse_tensor.dense_shape) + dense_shape=input_sparse_tensor.dense_shape, + ) # Compute the third dimension explicitly instead of setting it to -1, as # that doesn't work for dynamically shaped tensors with 0-length at runtime. # This happens for empty sequences. @@ -3082,16 +3119,17 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class SequenceNumericColumn( DenseColumn, CategoricalColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._CategoricalColumn, # pylint: disable=protected-access + fc_old._DenseColumn, + fc_old._CategoricalColumn, collections.namedtuple('SequenceNumericColumn', - ('source_column', 'sequence_length'))): + ('source_column', 'sequence_length')), +): """See `SequenceNumericColumn`.""" @property def _is_v2_column(self): - return (isinstance(self.source_column, FeatureColumn) and - self.source_column._is_v2_column) # pylint: disable=protected-access + return isinstance(self.source_column, + FeatureColumn) and self.source_column._is_v2_column @property def name(self): @@ -3112,7 +3150,7 @@ def parse_example_spec(self): @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - return self.source_column._parse_example_spec # pylint: disable=protected-access + return self.source_column._parse_example_spec @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) @@ -3142,8 +3180,9 @@ def _get_dense_tensor_for_input_tensor(self, input_tensor): return array_ops.one_hot( indices=math_ops.cast(input_tensor, dtypes.int64), depth=self.sequence_length, - on_value=1., - off_value=0.) + on_value=1.0, + off_value=0.0, + ) def get_dense_tensor(self, transformation_cache, state_manager): """Returns one hot encoded dense `Tensor`.""" @@ -3232,16 +3271,18 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class SequenceWeightedCategoricalColumn( CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access + fc_old._CategoricalColumn, collections.namedtuple( 'SequenceWeightedCategoricalColumn', - ('categorical_column', 'weight_feature_key', 'dtype'))): + ('categorical_column', 'weight_feature_key', 'dtype'), + ), +): """See `weighted_categorical_column`.""" @property def _is_v2_column(self): - return (isinstance(self.categorical_column, FeatureColumn) and - self.categorical_column._is_v2_column) # pylint: disable=protected-access + return isinstance(self.categorical_column, + FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): @@ -3268,7 +3309,7 @@ def parse_example_spec(self): @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - config = self.categorical_column._parse_example_spec # pylint: disable=protected-access + config = self.categorical_column._parse_example_spec if self.weight_feature_key in config: raise ValueError('Parse config {} already exists for {}.'.format( config[self.weight_feature_key], self.weight_feature_key)) @@ -3284,7 +3325,7 @@ def num_buckets(self): @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): - return self.categorical_column._num_buckets # pylint: disable=protected-access + return self.categorical_column._num_buckets def _transform_weight_tensor(self, weight_tensor): if weight_tensor is None: @@ -3310,8 +3351,10 @@ def transform_feature(self, transformation_cache, state_manager): weight_tensor = transformation_cache.get(self.weight_feature_key, state_manager) weight_tensor = self._transform_weight_tensor(weight_tensor) - return (transformation_cache.get(self.categorical_column, - state_manager), weight_tensor) + return ( + transformation_cache.get(self.categorical_column, state_manager), + weight_tensor, + ) @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) @@ -3364,19 +3407,30 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class EmbeddingColumn( DenseColumn, SequenceDenseColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._SequenceDenseColumn, # pylint: disable=protected-access + fc_old._DenseColumn, + fc_old._SequenceDenseColumn, collections.namedtuple( 'EmbeddingColumn', - ('categorical_column', 'dimension', 'combiner', 'initializer', - 'ckpt_to_load_from', 'tensor_name_in_ckpt', 'max_norm', 'trainable', - 'partitioner', 'ev_params'))): + ( + 'categorical_column', + 'dimension', + 'combiner', + 'initializer', + 'ckpt_to_load_from', + 'tensor_name_in_ckpt', + 'max_norm', + 'trainable', + 'partitioner', + 'ev_params', + ), + ), +): """See `embedding_column`.""" @property def _is_v2_column(self): - return (isinstance(self.categorical_column, FeatureColumn) and - self.categorical_column._is_v2_column) # pylint: disable=protected-access + return isinstance(self.categorical_column, + FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): @@ -3397,7 +3451,7 @@ def parse_example_spec(self): @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - return self.categorical_column._parse_example_spec # pylint: disable=protected-access + return self.categorical_column._parse_example_spec def transform_feature(self, transformation_cache, state_manager): """Transforms underlying `categorical_column`.""" @@ -3421,7 +3475,7 @@ def _variable_shape(self): def create_state(self, state_manager): """Creates the embedding lookup variable.""" - embedding_shape = (self.categorical_column._num_buckets, self.dimension) # pylint: disable=protected-access + embedding_shape = (self.categorical_column._num_buckets, self.dimension) state_manager.create_variable( self, name='embedding_weights', @@ -3429,7 +3483,8 @@ def create_state(self, state_manager): dtype=dtypes.float32, trainable=self.trainable, use_resource=True, - initializer=self.initializer) + initializer=self.initializer, + ) def _get_dense_tensor_internal_helper(self, sparse_tensors, embedding_weights): @@ -3439,7 +3494,7 @@ def _get_dense_tensor_internal_helper(self, sparse_tensors, if self.ckpt_to_load_from is not None: to_restore = embedding_weights if isinstance(to_restore, variables.PartitionedVariable): - to_restore = to_restore._get_variable_list() # pylint: disable=protected-access + to_restore = to_restore._get_variable_list() checkpoint_utils.init_from_checkpoint( self.ckpt_to_load_from, {self.tensor_name_in_ckpt: to_restore}) @@ -3450,7 +3505,8 @@ def _get_dense_tensor_internal_helper(self, sparse_tensors, sparse_weights, combiner=self.combiner, max_norm=self.max_norm, - name='%s_weights' % self.name) + name='%s_weights' % self.name, + ) # Return embedding lookup result. return embedding_lookup_sparse( @@ -3459,7 +3515,8 @@ def _get_dense_tensor_internal_helper(self, sparse_tensors, sparse_weights, combiner=self.combiner, name='%s_weights' % self.name, - max_norm=self.max_norm) + max_norm=self.max_norm, + ) def _get_dense_tensor_internal(self, sparse_tensors, state_manager): """Private method that follows the signature of get_dense_tensor.""" @@ -3471,9 +3528,8 @@ def _get_dense_tensor_internal(self, sparse_tensors, state_manager): def _old_get_dense_tensor_internal(self, sparse_tensors, weight_collections, trainable): """Private method that follows the signature of _get_dense_tensor.""" - embedding_shape = (self.categorical_column._num_buckets, self.dimension) # pylint: disable=protected-access - if (weight_collections and - ops.GraphKeys.GLOBAL_VARIABLES not in weight_collections): + embedding_shape = (self.categorical_column._num_buckets, self.dimension) + if weight_collections and ops.GraphKeys.GLOBAL_VARIABLES not in weight_collections: weight_collections.append(ops.GraphKeys.GLOBAL_VARIABLES) if self.ev_params is None: embedding_weights = variable_scope.get_variable( @@ -3483,13 +3539,14 @@ def _old_get_dense_tensor_internal(self, sparse_tensors, weight_collections, initializer=self.initializer, trainable=self.trainable and trainable, partitioner=self.partitioner, - collections=weight_collections) + collections=weight_collections, + ) else: # at eval or inference time, it is necessary to set # the initializers to zeros, so that new key will # get zero embedding - if os.environ.get('tf.estimator.mode', '') != \ - os.environ.get('tf.estimator.ModeKeys.TRAIN', 'train'): + if os.environ.get('tf.estimator.mode', '') != os.environ.get( + 'tf.estimator.ModeKeys.TRAIN', 'train'): initializer = init_ops.zeros_initializer() else: initializer = self.initializer @@ -3510,7 +3567,8 @@ def _old_get_dense_tensor_internal(self, sparse_tensors, weight_collections, partitioner=self.partitioner, collections=weight_collections, steps_to_live=self.ev_params.steps_to_live, - **extra_args) + **extra_args, + ) # Write the embedding configuration to RTP-specified collections. This will inform RTP to # optimize this embedding operation. @@ -3519,7 +3577,8 @@ def _old_get_dense_tensor_internal(self, sparse_tensors, weight_collections, variable=embedding_weights, bucket_size=self.categorical_column._num_buckets, combiner=self.combiner, - is_embedding_var=(self.ev_params is not None)) + is_embedding_var=(self.ev_params is not None), + ) embedding_attrs['name'] = layer_utils.unique_name_in_collection( compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs['name']) layer_utils.update_attr_to_collection( @@ -3534,11 +3593,17 @@ def _old_get_dense_tensor_internal(self, sparse_tensors, weight_collections, # the embedding subgraph. if isinstance(sparse_tensors.id_tensor, sparse_tensor_lib.SparseTensor): layer_utils.append_tensor_to_collection( - compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs['name'], - 'tensor', predictions) + compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, + embedding_attrs['name'], + 'tensor', + predictions, + ) layer_utils.append_tensor_to_collection( - compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, embedding_attrs['name'], - 'input', sparse_tensors.id_tensor) + compat_ops.GraphKeys.RANK_SERVICE_EMBEDDING, + embedding_attrs['name'], + 'input', + sparse_tensors.id_tensor, + ) return predictions @@ -3577,7 +3642,8 @@ def get_dense_tensor(self, transformation_cache, state_manager): def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): if isinstance( self.categorical_column, - (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn)): # pylint: disable=protected-access + (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn), + ): raise ValueError( 'In embedding_column: {}. ' 'categorical_column must not be of type _SequenceCategoricalColumn. ' @@ -3587,7 +3653,7 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): 'SequenceFeatures instead of DenseFeatures. ' 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column)) - sparse_tensors = self.categorical_column._get_sparse_tensors( # pylint: disable=protected-access + sparse_tensors = self.categorical_column._get_sparse_tensors( inputs, weight_collections, trainable) return self._old_get_dense_tensor_internal(sparse_tensors, weight_collections, trainable) @@ -3619,9 +3685,14 @@ def _get_sequence_dense_tensor(self, trainable=None): if not isinstance( self.categorical_column, - (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn, - SequenceBucketizedColumn, SequenceNumericColumn, - SequenceWeightedCategoricalColumn)): # pylint: disable=protected-access + ( + SequenceCategoricalColumn, + fc_old._SequenceCategoricalColumn, + SequenceBucketizedColumn, + SequenceNumericColumn, + SequenceWeightedCategoricalColumn, + ), + ): raise ValueError( 'In embedding_column: {}. ' 'categorical_column must be of type SequenceCategoricalColumn ' @@ -3629,7 +3700,7 @@ def _get_sequence_dense_tensor(self, 'Suggested fix: Use one of sequence_categorical_column_with_*. ' 'Given (type {}): {}'.format(self.name, type(self.categorical_column), self.categorical_column)) - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) dense_tensor = self._old_get_dense_tensor_internal( sparse_tensors, weight_collections=weight_collections, @@ -3673,14 +3744,16 @@ def _raise_shared_embedding_column_error(): # class SharedEmbeddingColumnCreator(tracking.AutoTrackable): class SharedEmbeddingColumnCreator: - def __init__(self, - dimension, - initializer, - ckpt_to_load_from, - tensor_name_in_ckpt, - num_buckets, - trainable, - name='shared_embedding_column_creator'): + def __init__( + self, + dimension, + initializer, + ckpt_to_load_from, + tensor_name_in_ckpt, + num_buckets, + trainable, + name='shared_embedding_column_creator', + ): self._dimension = dimension self._initializer = initializer self._ckpt_to_load_from = ckpt_to_load_from @@ -3696,7 +3769,7 @@ def __call__(self, categorical_column, combiner, max_norm): @property def embedding_weights(self): - key = ops.get_default_graph()._graph_key # pylint: disable=protected-access + key = ops.get_default_graph()._graph_key if key not in self._embedding_weights: embedding_shape = (self._num_buckets, self._dimension) var = variable_scope.get_variable( @@ -3704,12 +3777,13 @@ def embedding_weights(self): shape=embedding_shape, dtype=dtypes.float32, initializer=self._initializer, - trainable=self._trainable) + trainable=self._trainable, + ) if self._ckpt_to_load_from is not None: to_restore = var if isinstance(to_restore, variables.PartitionedVariable): - to_restore = to_restore._get_variable_list() # pylint: disable=protected-access + to_restore = to_restore._get_variable_list() checkpoint_utils.init_from_checkpoint( self._ckpt_to_load_from, {self._tensor_name_in_ckpt: to_restore}) self._embedding_weights[key] = var @@ -3723,12 +3797,18 @@ def dimension(self): class SharedEmbeddingColumn( DenseColumn, SequenceDenseColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._SequenceDenseColumn, # pylint: disable=protected-access + fc_old._DenseColumn, + fc_old._SequenceDenseColumn, collections.namedtuple( 'SharedEmbeddingColumn', - ('categorical_column', 'shared_embedding_column_creator', 'combiner', - 'max_norm'))): + ( + 'categorical_column', + 'shared_embedding_column_creator', + 'combiner', + 'max_norm', + ), + ), +): """See `embedding_column`.""" @property @@ -3792,7 +3872,8 @@ def _get_dense_tensor_internal(self, transformation_cache, state_manager): sparse_weights=sparse_weights, combiner=self.combiner, name='%s_weights' % self.name, - max_norm=self.max_norm) + max_norm=self.max_norm, + ) def get_dense_tensor(self, transformation_cache, state_manager): """Returns the embedding lookup result.""" @@ -3869,11 +3950,12 @@ def _check_shape(shape, key): class HashedCategoricalColumn( CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple('HashedCategoricalColumn', - ('feature_name', 'key', 'hash_bucket_size', 'dtype')) + fc_old._CategoricalColumn, + collections.namedtuple( + 'HashedCategoricalColumn', + ('feature_name', 'key', 'hash_bucket_size', 'dtype')), ): - """see `categorical_column_with_hash_bucket`.""" + """See `categorical_column_with_hash_bucket`.""" @property def _is_v2_column(self): @@ -3922,6 +4004,7 @@ def _transform_input_tensor(self, input_tensor): if 'RaggedTensor' in str(type(input_tensor)): from tensorflow.python.ops.ragged import ragged_tensor + return ragged_tensor.RaggedTensor.from_row_splits( values=sparse_id_values, row_splits=input_tensor.row_splits) @@ -3989,11 +4072,20 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class VocabularyFileCategoricalColumn( CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access + fc_old._CategoricalColumn, collections.namedtuple( 'VocabularyFileCategoricalColumn', - ('feature_name', 'key', 'vocabulary_file', 'vocabulary_size', - 'num_oov_buckets', 'dtype', 'default_value'))): + ( + 'feature_name', + 'key', + 'vocabulary_file', + 'vocabulary_size', + 'num_oov_buckets', + 'dtype', + 'default_value', + ), + ), +): """See `categorical_column_with_vocabulary_file`.""" @property @@ -4046,7 +4138,8 @@ def _transform_input_tensor(self, input_tensor): vocab_size=self.vocabulary_size, default_value=self.default_value, key_dtype=key_dtype, - name='{}_lookup'.format(self.key)).lookup(input_tensor) + name='{}_lookup'.format(self.key), + ).lookup(input_tensor) def transform_feature(self, transformation_cache, state_manager): """Creates a lookup table for the vocabulary.""" @@ -4108,10 +4201,19 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class VocabularyListCategoricalColumn( CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access - collections.namedtuple('VocabularyListCategoricalColumn', - ('feature_name', 'key', 'vocabulary_list', 'dtype', - 'default_value', 'num_oov_buckets'))): + fc_old._CategoricalColumn, + collections.namedtuple( + 'VocabularyListCategoricalColumn', + ( + 'feature_name', + 'key', + 'vocabulary_list', + 'dtype', + 'default_value', + 'num_oov_buckets', + ), + ), +): """See `categorical_column_with_vocabulary_list`.""" @property @@ -4163,7 +4265,8 @@ def _transform_input_tensor(self, input_tensor): default_value=self.default_value, num_oov_buckets=self.num_oov_buckets, dtype=key_dtype, - name='{}_lookup'.format(self.key)).lookup(input_tensor) + name='{}_lookup'.format(self.key), + ).lookup(input_tensor) def transform_feature(self, transformation_cache, state_manager): """Creates a lookup table for the vocabulary list.""" @@ -4225,10 +4328,12 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class IdentityCategoricalColumn( CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access + fc_old._CategoricalColumn, collections.namedtuple( 'IdentityCategoricalColumn', - ('feature_name', 'key', 'number_buckets', 'default_value'))): + ('feature_name', 'key', 'number_buckets', 'default_value'), + ), +): """See `categorical_column_with_identity`.""" @property @@ -4276,7 +4381,8 @@ def _transform_input_tensor(self, input_tensor): values, num_buckets, data=(values, num_buckets), - name='assert_less_than_num_buckets') + name='assert_less_than_num_buckets', + ) assert_greater = check_ops.assert_greater_equal( values, zero, data=(values,), name='assert_greater_or_equal_0') with ops.control_dependencies((assert_less, assert_greater)): @@ -4289,12 +4395,16 @@ def _transform_input_tensor(self, input_tensor): array_ops.fill( dims=array_ops.shape(values), value=math_ops.cast(self.default_value, dtypes.int64), - name='default_values'), values) + name='default_values', + ), + values, + ) return sparse_tensor_lib.SparseTensor( indices=input_tensor.indices, values=values, - dense_shape=input_tensor.dense_shape) + dense_shape=input_tensor.dense_shape, + ) def transform_feature(self, transformation_cache, state_manager): """Returns a SparseTensor with identity values.""" @@ -4352,16 +4462,18 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class WeightedCategoricalColumn( CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access + fc_old._CategoricalColumn, collections.namedtuple( 'WeightedCategoricalColumn', - ('categorical_column', 'weight_feature_key', 'dtype'))): + ('categorical_column', 'weight_feature_key', 'dtype'), + ), +): """See `weighted_categorical_column`.""" @property def _is_v2_column(self): - return (isinstance(self.categorical_column, FeatureColumn) and - self.categorical_column._is_v2_column) # pylint: disable=protected-access + return isinstance(self.categorical_column, + FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): @@ -4388,7 +4500,7 @@ def parse_example_spec(self): @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - config = self.categorical_column._parse_example_spec # pylint: disable=protected-access + config = self.categorical_column._parse_example_spec if self.weight_feature_key in config: raise ValueError('Parse config {} already exists for {}.'.format( config[self.weight_feature_key], self.weight_feature_key)) @@ -4404,7 +4516,7 @@ def num_buckets(self): @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): - return self.categorical_column._num_buckets # pylint: disable=protected-access + return self.categorical_column._num_buckets def _transform_weight_tensor(self, weight_tensor): if weight_tensor is None: @@ -4427,8 +4539,10 @@ def transform_feature(self, transformation_cache, state_manager): weight_tensor = transformation_cache.get(self.weight_feature_key, state_manager) weight_tensor = self._transform_weight_tensor(weight_tensor) - return (transformation_cache.get(self.categorical_column, - state_manager), weight_tensor) + return ( + transformation_cache.get(self.categorical_column, state_manager), + weight_tensor, + ) @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) @@ -4480,10 +4594,11 @@ def _from_config(cls, config, custom_objects=None, columns_by_name=None): class CrossedColumn( CategoricalColumn, - fc_old._CategoricalColumn, # pylint: disable=protected-access + fc_old._CategoricalColumn, collections.namedtuple( 'CrossedColumn', - ('feature_name', 'keys', 'hash_bucket_size', 'hash_key'))): + ('feature_name', 'keys', 'hash_bucket_size', 'hash_key')), +): """See `crossed_column`.""" @property @@ -4493,7 +4608,7 @@ def _is_v2_column(self): continue if not isinstance(key, FeatureColumn): return False - if not key._is_v2_column: # pylint: disable=protected-access + if not key._is_v2_column: return False return True @@ -4504,7 +4619,7 @@ def name(self): return self.feature_name feature_names = [] for key in _collect_leaf_level_keys(self): - if isinstance(key, (FeatureColumn, fc_old._FeatureColumn)): # pylint: disable=protected-access + if isinstance(key, (FeatureColumn, fc_old._FeatureColumn)): feature_names.append(key.name) else: # key must be a string feature_names.append(key) @@ -4517,8 +4632,8 @@ def parse_example_spec(self): for key in self.keys: if isinstance(key, FeatureColumn): config.update(key.parse_example_spec) - elif isinstance(key, fc_old._FeatureColumn): # pylint: disable=protected-access - config.update(key._parse_example_spec) # pylint: disable=protected-access + elif isinstance(key, fc_old._FeatureColumn): + config.update(key._parse_example_spec) else: # key must be a string config.update({key: parsing_ops.VarLenFeature(dtypes.string)}) return config @@ -4535,7 +4650,7 @@ def transform_feature(self, transformation_cache, state_manager): for key in _collect_leaf_level_keys(self): if isinstance(key, six.string_types): feature_tensors.append(transformation_cache.get(key, state_manager)) - elif isinstance(key, (fc_old._CategoricalColumn, CategoricalColumn)): # pylint: disable=protected-access + elif isinstance(key, (fc_old._CategoricalColumn, CategoricalColumn)): ids_and_weights = key.get_sparse_tensors(transformation_cache, state_manager) if ids_and_weights.weight_tensor is not None: @@ -4549,7 +4664,8 @@ def transform_feature(self, transformation_cache, state_manager): return sparse_ops.sparse_cross_hashed( inputs=feature_tensors, num_buckets=self.hash_bucket_size, - hash_key=self.hash_key) + hash_key=self.hash_key, + ) @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) @@ -4559,8 +4675,8 @@ def _transform_feature(self, inputs): for key in _collect_leaf_level_keys(self): if isinstance(key, six.string_types): feature_tensors.append(inputs.get(key)) - elif isinstance(key, (CategoricalColumn, fc_old._CategoricalColumn)): # pylint: disable=protected-access - ids_and_weights = key._get_sparse_tensors(inputs) # pylint: disable=protected-access + elif isinstance(key, (CategoricalColumn, fc_old._CategoricalColumn)): + ids_and_weights = key._get_sparse_tensors(inputs) if ids_and_weights.weight_tensor is not None: raise ValueError( 'crossed_column does not support weight_tensor, but the given ' @@ -4572,7 +4688,8 @@ def _transform_feature(self, inputs): return sparse_ops.sparse_cross_hashed( inputs=feature_tensors, num_buckets=self.hash_bucket_size, - hash_key=self.hash_key) + hash_key=self.hash_key, + ) @property def num_buckets(self): @@ -4667,9 +4784,10 @@ def _prune_invalid_weights(sparse_ids, sparse_weights): class IndicatorColumn( DenseColumn, SequenceDenseColumn, - fc_old._DenseColumn, # pylint: disable=protected-access - fc_old._SequenceDenseColumn, # pylint: disable=protected-access - collections.namedtuple('IndicatorColumn', ('categorical_column'))): + fc_old._DenseColumn, + fc_old._SequenceDenseColumn, + collections.namedtuple('IndicatorColumn', ('categorical_column')), +): """Represents a one-hot column for use in deep networks. Args: @@ -4679,8 +4797,8 @@ class IndicatorColumn( @property def _is_v2_column(self): - return (isinstance(self.categorical_column, FeatureColumn) and - self.categorical_column._is_v2_column) # pylint: disable=protected-access + return isinstance(self.categorical_column, + FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): @@ -4701,15 +4819,18 @@ def _transform_id_weight_pair(self, id_weight_pair): weighted_column = sparse_ops.sparse_merge( sp_ids=id_tensor, sp_values=weight_tensor, - vocab_size=int(self._variable_shape[-1])) + vocab_size=int(self._variable_shape[-1]), + ) # Remove (?, -1) index. weighted_column = sparse_ops.sparse_slice(weighted_column, [0, 0], weighted_column.dense_shape) # Use scatter_nd to merge duplicated indices if existed, # instead of sparse_tensor_to_dense. - return array_ops.scatter_nd(weighted_column.indices, - weighted_column.values, - weighted_column.dense_shape) + return array_ops.scatter_nd( + weighted_column.indices, + weighted_column.values, + weighted_column.dense_shape, + ) dense_id_tensor = sparse_ops.sparse_tensor_to_dense( id_tensor, default_value=-1) @@ -4747,7 +4868,7 @@ def transform_feature(self, transformation_cache, state_manager): @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _transform_feature(self, inputs): - id_weight_pair = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access + id_weight_pair = self.categorical_column._get_sparse_tensors(inputs) return self._transform_id_weight_pair(id_weight_pair) @property @@ -4759,7 +4880,7 @@ def parse_example_spec(self): @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - return self.categorical_column._parse_example_spec # pylint: disable=protected-access + return self.categorical_column._parse_example_spec @property def variable_shape(self): @@ -4767,13 +4888,13 @@ def variable_shape(self): if isinstance(self.categorical_column, FeatureColumn): return tensor_shape.TensorShape([1, self.categorical_column.num_buckets]) else: - return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) # pylint: disable=protected-access + return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) @property @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _variable_shape(self): - return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) # pylint: disable=protected-access + return tensor_shape.TensorShape([1, self.categorical_column._num_buckets]) def get_dense_tensor(self, transformation_cache, state_manager): """Returns dense `Tensor` representing feature. @@ -4811,7 +4932,8 @@ def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None): del trainable if isinstance( self.categorical_column, - (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn)): # pylint: disable=protected-access + (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn), + ): raise ValueError( 'In indicator_column: {}. ' 'categorical_column must not be of type _SequenceCategoricalColumn. ' @@ -4857,7 +4979,8 @@ def _get_sequence_dense_tensor(self, del trainable if not isinstance( self.categorical_column, - (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn)): # pylint: disable=protected-access + (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn), + ): raise ValueError( 'In indicator_column: {}. ' 'categorical_column must be of type _SequenceCategoricalColumn ' @@ -4868,7 +4991,7 @@ def _get_sequence_dense_tensor(self, # Feature has been already transformed. Return the intermediate # representation created by _transform_feature. dense_tensor = inputs.get(self) - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) sequence_length = fc_utils.sequence_length_from_sparse_tensor( sparse_tensors.id_tensor) return SequenceDenseColumn.TensorSequenceLengthPair( @@ -4919,21 +5042,24 @@ def _verify_static_batch_size_equality(tensors, columns): raise ValueError( 'Batch size (first dimension) of each feature must be same. ' 'Batch size of columns ({}, {}): ({}, {})'.format( - columns[bath_size_column_index].name, columns[i].name, - expected_batch_size, batch_size)) + columns[bath_size_column_index].name, + columns[i].name, + expected_batch_size, + batch_size, + )) class SequenceCategoricalColumn( CategoricalColumn, - fc_old._SequenceCategoricalColumn, # pylint: disable=protected-access - collections.namedtuple('SequenceCategoricalColumn', - ('categorical_column'))): + fc_old._SequenceCategoricalColumn, + collections.namedtuple('SequenceCategoricalColumn', ('categorical_column')), +): """Represents sequences of categorical data.""" @property def _is_v2_column(self): - return (isinstance(self.categorical_column, FeatureColumn) and - self.categorical_column._is_v2_column) # pylint: disable=protected-access + return isinstance(self.categorical_column, + FeatureColumn) and self.categorical_column._is_v2_column @property def name(self): @@ -4954,7 +5080,7 @@ def parse_example_spec(self): @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _parse_example_spec(self): - return self.categorical_column._parse_example_spec # pylint: disable=protected-access + return self.categorical_column._parse_example_spec def transform_feature(self, transformation_cache, state_manager): """See `FeatureColumn` base class.""" @@ -4983,7 +5109,7 @@ def num_buckets(self): @deprecation.deprecated(_FEATURE_COLUMN_DEPRECATION_DATE, _FEATURE_COLUMN_DEPRECATION) def _num_buckets(self): - return self.categorical_column._num_buckets # pylint: disable=protected-access + return self.categorical_column._num_buckets def _get_sparse_tensors_helper(self, sparse_tensors): id_tensor = sparse_tensors.id_tensor @@ -5030,7 +5156,7 @@ def _get_sparse_tensors(self, inputs, weight_collections=None, trainable=None): - sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) # pylint: disable=protected-access + sparse_tensors = self.categorical_column._get_sparse_tensors(inputs) return self._get_sparse_tensors_helper(sparse_tensors) @property @@ -5145,11 +5271,19 @@ def deserialize_feature_column(config, # FeatureColumns not part of the module can be passed as custom_objects. module_feature_column_classes = { cls.__name__: cls for cls in [ - BucketizedColumn, EmbeddingColumn, HashedCategoricalColumn, - IdentityCategoricalColumn, IndicatorColumn, NumericColumn, - SequenceCategoricalColumn, SequenceDenseColumn, SharedEmbeddingColumn, - VocabularyFileCategoricalColumn, VocabularyListCategoricalColumn, - WeightedCategoricalColumn, init_ops.TruncatedNormal + BucketizedColumn, + EmbeddingColumn, + HashedCategoricalColumn, + IdentityCategoricalColumn, + IndicatorColumn, + NumericColumn, + SequenceCategoricalColumn, + SequenceDenseColumn, + SharedEmbeddingColumn, + VocabularyFileCategoricalColumn, + VocabularyListCategoricalColumn, + WeightedCategoricalColumn, + init_ops.TruncatedNormal, ] } if columns_by_name is None: @@ -5159,14 +5293,15 @@ def deserialize_feature_column(config, config, module_objects=module_feature_column_classes, custom_objects=custom_objects, - printable_module_name='feature_column_v2') + printable_module_name='feature_column_v2', + ) if not issubclass(cls, FeatureColumn): raise ValueError( 'Expected FeatureColumn class, instead found: {}'.format(cls)) # Always deserialize the FeatureColumn, in order to get the name. - new_instance = cls._from_config( # pylint: disable=protected-access + new_instance = cls._from_config( cls_config, custom_objects=custom_objects, columns_by_name=columns_by_name) diff --git a/easy_rec/python/compat/feature_column/sequence_feature_column.py b/easy_rec/python/compat/feature_column/sequence_feature_column.py index b0fcdc9f7..2ac498d81 100644 --- a/easy_rec/python/compat/feature_column/sequence_feature_column.py +++ b/easy_rec/python/compat/feature_column/sequence_feature_column.py @@ -27,16 +27,15 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import check_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import parsing_ops -from tensorflow.python.ops import sparse_ops from easy_rec.python.compat.feature_column import feature_column as fc_v1 from easy_rec.python.compat.feature_column import feature_column_v2 as fc from easy_rec.python.compat.feature_column import utils as fc_utils +from tensorflow.python.ops import ( # NOQA + array_ops, check_ops, math_ops, parsing_ops, sparse_ops, +) + # pylint: disable=protected-access @@ -96,7 +95,8 @@ def __init__(self, feature_columns, trainable=True, name=None, **kwargs): trainable=trainable, name=name, expected_column_type=fc.SequenceDenseColumn, - **kwargs) + **kwargs, + ) def _target_shape(self, input_shape, total_elements): return (input_shape[0], input_shape[1], total_elements) @@ -166,28 +166,33 @@ def concatenate_context_input(context_input, sequence_input): sequence_input, 3, message='sequence_input must have rank 3', - data=[array_ops.shape(sequence_input)]) + data=[array_ops.shape(sequence_input)], + ) seq_type_check = check_ops.assert_type( sequence_input, dtypes.float32, message='sequence_input must have dtype float32; got {}.'.format( - sequence_input.dtype)) + sequence_input.dtype), + ) ctx_rank_check = check_ops.assert_rank( context_input, 2, message='context_input must have rank 2', - data=[array_ops.shape(context_input)]) + data=[array_ops.shape(context_input)], + ) ctx_type_check = check_ops.assert_type( context_input, dtypes.float32, message='context_input must have dtype float32; got {}.'.format( - context_input.dtype)) + context_input.dtype), + ) with ops.control_dependencies( [seq_rank_check, seq_type_check, ctx_rank_check, ctx_type_check]): padded_length = array_ops.shape(sequence_input)[1] tiled_context_input = array_ops.tile( array_ops.expand_dims(context_input, 1), - array_ops.concat([[1], [padded_length], [1]], 0)) + array_ops.concat([[1], [padded_length], [1]], 0), + ) return array_ops.concat([sequence_input, tiled_context_input], 2) @@ -239,7 +244,8 @@ def sequence_categorical_column_with_identity(key, feature_name=feature_name, key=key, num_buckets=num_buckets, - default_value=default_value)) + default_value=default_value, + )) def sequence_numeric_column_with_bucketized_column(source_column, boundaries): @@ -280,7 +286,8 @@ def sequence_weighted_categorical_column(categorical_column, return fc.SequenceWeightedCategoricalColumn( categorical_column=categorical_column, weight_feature_key=weight_feature_key, - dtype=dtype) + dtype=dtype, + ) def sequence_categorical_column_with_hash_bucket(key, @@ -328,16 +335,19 @@ def sequence_categorical_column_with_hash_bucket(key, feature_name=feature_name, key=key, hash_bucket_size=hash_bucket_size, - dtype=dtype)) - - -def sequence_categorical_column_with_vocabulary_file(key, - vocabulary_file, - vocabulary_size=None, - num_oov_buckets=0, - default_value=None, - dtype=dtypes.string, - feature_name=None): + dtype=dtype, + )) + + +def sequence_categorical_column_with_vocabulary_file( + key, + vocabulary_file, + vocabulary_size=None, + num_oov_buckets=0, + default_value=None, + dtype=dtypes.string, + feature_name=None, +): """A sequence of categorical terms where ids use a vocabulary file. Pass this to `embedding_column` or `indicator_column` to convert sequence @@ -397,15 +407,18 @@ def sequence_categorical_column_with_vocabulary_file(key, vocabulary_size=vocabulary_size, num_oov_buckets=num_oov_buckets, default_value=default_value, - dtype=dtype)) + dtype=dtype, + )) -def sequence_categorical_column_with_vocabulary_list(key, - vocabulary_list, - dtype=None, - default_value=-1, - num_oov_buckets=0, - feature_name=None): +def sequence_categorical_column_with_vocabulary_list( + key, + vocabulary_list, + dtype=None, + default_value=-1, + num_oov_buckets=0, + feature_name=None, +): """A sequence of categorical terms where ids use an in-memory list. Pass this to `embedding_column` or `indicator_column` to convert sequence @@ -463,15 +476,18 @@ def sequence_categorical_column_with_vocabulary_list(key, vocabulary_list=vocabulary_list, dtype=dtype, default_value=default_value, - num_oov_buckets=num_oov_buckets)) + num_oov_buckets=num_oov_buckets, + )) -def sequence_numeric_column(key, - shape=(1,), - default_value=0., - dtype=dtypes.float32, - normalizer_fn=None, - feature_name=None): +def sequence_numeric_column( + key, + shape=(1,), + default_value=0.0, + dtype=dtypes.float32, + normalizer_fn=None, + feature_name=None, +): """Returns a feature column that represents sequences of numeric data. Example: @@ -526,7 +542,8 @@ def sequence_numeric_column(key, shape=shape, default_value=default_value, dtype=dtype, - normalizer_fn=normalizer_fn) + normalizer_fn=normalizer_fn, + ) def _assert_all_equal_and_return(tensors, name=None): @@ -542,10 +559,14 @@ def _assert_all_equal_and_return(tensors, name=None): class SequenceNumericColumn( - fc.SequenceDenseColumn, fc_v1._FeatureColumn, - collections.namedtuple('SequenceNumericColumn', - ('feature_name', 'key', 'shape', 'default_value', - 'dtype', 'normalizer_fn'))): + fc.SequenceDenseColumn, + fc_v1._FeatureColumn, + collections.namedtuple( + 'SequenceNumericColumn', + ('feature_name', 'key', 'shape', 'default_value', 'dtype', + 'normalizer_fn'), + ), +): """Represents sequences of numeric data.""" @property diff --git a/easy_rec/python/compat/layers.py b/easy_rec/python/compat/layers.py index 651eefac8..3eb4fd17c 100644 --- a/easy_rec/python/compat/layers.py +++ b/easy_rec/python/compat/layers.py @@ -28,17 +28,19 @@ from tensorflow.python.ops import variable_scope -def layer_norm(inputs, - center=True, - scale=True, - activation_fn=None, - reuse=None, - variables_collections=None, - outputs_collections=None, - trainable=True, - begin_norm_axis=1, - begin_params_axis=-1, - scope=None): +def layer_norm( + inputs, + center=True, + scale=True, + activation_fn=None, + reuse=None, + variables_collections=None, + outputs_collections=None, + trainable=True, + begin_norm_axis=1, + begin_params_axis=-1, + scope=None, +): """Adds a Layer Normalization layer. Based on the paper: @@ -126,7 +128,8 @@ def layer_norm(inputs, dtype=dtype, initializer=init_ops.zeros_initializer(), collections=beta_collections, - trainable=trainable) + trainable=trainable, + ) if scale: gamma_collections = get_variable_collections(variables_collections, 'gamma') @@ -136,7 +139,8 @@ def layer_norm(inputs, dtype=dtype, initializer=init_ops.ones_initializer(), collections=gamma_collections, - trainable=trainable) + trainable=trainable, + ) # Calculate the moments on the last axis (layer activations). norm_axes = list(range(begin_norm_axis, inputs_rank)) mean, variance = nn.moments(inputs, norm_axes, keep_dims=True) @@ -148,7 +152,8 @@ def layer_norm(inputs, variance, offset=beta, scale=gamma, - variance_epsilon=variance_epsilon) + variance_epsilon=variance_epsilon, + ) outputs.set_shape(inputs_shape) if activation_fn is not None: outputs = activation_fn(outputs) @@ -205,18 +210,20 @@ def append_tensor_alias(tensor, alias): return tensor -def variable(name, - shape=None, - dtype=None, - initializer=None, - regularizer=None, - trainable=True, - collections=None, - caching_device=None, - device=None, - partitioner=None, - custom_getter=None, - use_resource=None): +def variable( + name, + shape=None, + dtype=None, + initializer=None, + regularizer=None, + trainable=True, + collections=None, + caching_device=None, + device=None, + partitioner=None, + custom_getter=None, + use_resource=None, +): """Gets an existing variable with these parameters or creates a new one. Args: @@ -266,21 +273,24 @@ def variable(name, collections=collections, caching_device=caching_device, partitioner=partitioner, - use_resource=use_resource) - - -def model_variable(name, - shape=None, - dtype=dtypes.float32, - initializer=None, - regularizer=None, - trainable=True, - collections=None, - caching_device=None, - device=None, - partitioner=None, - custom_getter=None, - use_resource=None): + use_resource=use_resource, + ) + + +def model_variable( + name, + shape=None, + dtype=dtypes.float32, + initializer=None, + regularizer=None, + trainable=True, + collections=None, + caching_device=None, + device=None, + partitioner=None, + custom_getter=None, + use_resource=None, +): """Gets an existing model variable with these parameters or creates a new one. Args: @@ -325,5 +335,6 @@ def model_variable(name, device=device, partitioner=partitioner, custom_getter=custom_getter, - use_resource=use_resource) + use_resource=use_resource, + ) return var diff --git a/easy_rec/python/compat/optimizers.py b/easy_rec/python/compat/optimizers.py index d31a4cd41..ed3fadb7c 100644 --- a/easy_rec/python/compat/optimizers.py +++ b/easy_rec/python/compat/optimizers.py @@ -26,14 +26,6 @@ # from tensorflow.contrib import framework as contrib_framework from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -# from tensorflow.python.ops import logging_ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import gen_nn_ops -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import random_ops from tensorflow.python.ops import variable_scope as vs from tensorflow.python.ops import variables as vars_ from tensorflow.python.summary import summary @@ -45,6 +37,12 @@ from easy_rec.python.utils import constant from easy_rec.python.utils import estimator_utils +# from tensorflow.python.ops import logging_ops +from tensorflow.python.ops import ( # NOQA + array_ops, clip_ops, control_flow_ops, gen_nn_ops, init_ops, math_ops, + random_ops, +) + try: from tensorflow.python.framework import indexed_slices except Exception: @@ -57,6 +55,7 @@ try: from sparse_operation_kit import experiment as sok + from easy_rec.python.compat import sok_optimizer except Exception: sok = None @@ -70,7 +69,7 @@ train.FtrlOptimizer, 'Momentum': lambda learning_rate: train.MomentumOptimizer( - learning_rate, momentum=0.9), # pylint: disable=line-too-long + learning_rate, momentum=0.9), 'RMSProp': train.RMSPropOptimizer, 'SGD': @@ -86,23 +85,25 @@ ] -def optimize_loss(loss, - global_step, - learning_rate, - optimizer, - gradient_noise_scale=None, - gradient_multipliers=None, - clip_gradients=None, - learning_rate_decay_fn=None, - update_ops=None, - variables=None, - name=None, - summaries=None, - colocate_gradients_with_ops=False, - not_apply_grad_after_first_step=False, - increment_global_step=True, - incr_save=False, - embedding_parallel=False): +def optimize_loss( + loss, + global_step, + learning_rate, + optimizer, + gradient_noise_scale=None, + gradient_multipliers=None, + clip_gradients=None, + learning_rate_decay_fn=None, + update_ops=None, + variables=None, + name=None, + summaries=None, + colocate_gradients_with_ops=False, + not_apply_grad_after_first_step=False, + increment_global_step=True, + incr_save=False, + embedding_parallel=False, +): """Given loss and parameters for optimizer, returns a training op. Various ways of passing optimizers include: @@ -208,16 +209,18 @@ def optimize_loss(loss, # Learning rate variable, with possible decay. lr = None if learning_rate is not None: - if (isinstance(learning_rate, ops.Tensor) and - learning_rate.get_shape().ndims == 0): + if isinstance(learning_rate, + ops.Tensor) and learning_rate.get_shape().ndims == 0: lr = learning_rate elif isinstance(learning_rate, float): if learning_rate < 0.0: raise ValueError('Invalid learning_rate %s.', learning_rate) lr = vs.get_variable( - 'learning_rate', [], + 'learning_rate', + [], trainable=False, - initializer=init_ops.constant_initializer(learning_rate)) + initializer=init_ops.constant_initializer(learning_rate), + ) else: raise ValueError('Learning rate should be 0d Tensor or float. ' 'Got %s of type %s' % @@ -246,8 +249,8 @@ def optimize_loss(loss, 'Optimizer name should be one of [%s], you provided %s.' % (', '.join(OPTIMIZER_CLS_NAMES), optimizer)) opt = OPTIMIZER_CLS_NAMES[optimizer](learning_rate=lr) - elif (isinstance(optimizer, type) and - issubclass(optimizer, optimizer_.Optimizer)): + elif isinstance(optimizer, type) and issubclass(optimizer, + optimizer_.Optimizer): if lr is None: raise ValueError('Learning rate is None, but should be specified if ' 'optimizer is class (%s).' % optimizer) @@ -262,8 +265,8 @@ def optimize_loss(loss, if not isinstance(opt, optimizer_.Optimizer): raise ValueError('Unrecognized optimizer: function should return ' 'subclass of Optimizer. Got %s.' % str(opt)) - elif isinstance(optimizer, sok_optimizer.OptimizerWrapperV1) or \ - isinstance(optimizer, sok_optimizer.OptimizerWrapperV2): + elif isinstance(optimizer, sok_optimizer.OptimizerWrapperV1) or isinstance( + optimizer, sok_optimizer.OptimizerWrapperV2): opt = optimizer else: raise ValueError('Unrecognized optimizer: should be string, ' @@ -287,9 +290,14 @@ def optimize_loss(loss, # embedding parameters not partitioned reduced_grads = [] for g, v in gradients: - reduced_grads.append((hvd.allreduce( - g, op=hvd.Average, - compression=hvd.compression.NoneCompressor), v)) + reduced_grads.append(( + hvd.allreduce( + g, + op=hvd.Average, + compression=hvd.compression.NoneCompressor, + ), + v, + )) gradients = reduced_grads else: # embedding parameters partitioned: @@ -312,8 +320,11 @@ def optimize_loss(loss, part_grads.append(g) part_vars.append(v) else: - reduced_grads.append((indexed_slices.IndexedSlices( - indices=g.indices, values=g.values / hvd.size()), v)) + reduced_grads.append(( + indexed_slices.IndexedSlices( + indices=g.indices, values=g.values / hvd.size()), + v, + )) group_allreduce = False if len(part_grads) > 0: @@ -321,26 +332,34 @@ def optimize_loss(loss, reduced_part_grads = hvd.grouped_allreduce( part_grads, op=hvd.Average, - compression=hvd.compression.NoneCompressor) + compression=hvd.compression.NoneCompressor, + ) for g, v in zip(reduced_part_grads, part_vars): reduced_grads.append((g, v)) else: for g, v in zip(part_grads, part_vars): g = hvd.allreduce( - g, op=hvd.Average, compression=hvd.compression.NoneCompressor) + g, + op=hvd.Average, + compression=hvd.compression.NoneCompressor, + ) reduced_grads.append((g, v)) if len(part_sparse_grads) > 0: if group_allreduce: reduced_part_grads = hvd.grouped_allreduce( part_sparse_grads, op=hvd.Average, - compression=hvd.compression.NoneCompressor) + compression=hvd.compression.NoneCompressor, + ) for g, v in zip(reduced_part_grads, part_sparse_vars): reduced_grads.append((g, v)) else: for g, v in zip(part_sparse_grads, part_sparse_vars): g = hvd.allreduce( - g, op=hvd.Average, compression=hvd.compression.NoneCompressor) + g, + op=hvd.Average, + compression=hvd.compression.NoneCompressor, + ) reduced_grads.append((g, v)) gradients = reduced_grads @@ -397,8 +416,10 @@ def optimize_loss(loss, if 'gradients' in summaries: summary.histogram('gradients/%s' % var_name, grad_values) if 'gradient_norm' in summaries: - summary.scalar('gradient_norm/%s' % var_name, - clip_ops.global_norm([grad_values])) + summary.scalar( + 'gradient_norm/%s' % var_name, + clip_ops.global_norm([grad_values]), + ) if clip_gradients is not None and ('global_gradient_norm' in summaries or 'gradient_norm' in summaries): @@ -413,7 +434,8 @@ def _apply_grad(): grad_updates = opt.apply_gradients( gradients, global_step=global_step if increment_global_step else None, - name='train') + name='train', + ) embed_para_vars = ops.get_collection(constant.EmbeddingParallel) slot_names = opt.get_slot_names() @@ -464,16 +486,20 @@ def _get_grad_norm(grads_and_vars, embedding_parallel=False): sparse_norms.append(gen_nn_ops.l2_loss(grad.values)) else: dense_norms.append(gen_nn_ops.l2_loss(grad)) - reduced_norms = hvd.grouped_allreduce( - part_norms, op=hvd.Sum, compression=hvd.compression.NoneCompressor) - sparse_norms = sparse_norms + reduced_norms - all_norms = reduced_norms + dense_norms - sparse_norm = math_ops.sqrt( - math_ops.reduce_sum(array_ops.stack(sparse_norms) * 2.0)) - dense_norm = math_ops.sqrt( - math_ops.reduce_sum(array_ops.stack(dense_norms) * 2.0)) + if hvd is not None and part_norms: + reduced_norms = hvd.grouped_allreduce( + part_norms, op=hvd.Sum, compression=hvd.compression.NoneCompressor) + sparse_norms = sparse_norms + reduced_norms + all_norms = sparse_norms + dense_norms + sparse_norm = ( + math_ops.sqrt(math_ops.reduce_sum(array_ops.stack(sparse_norms) * 2.0)) + if sparse_norms else tf.constant(0.0)) + dense_norm = ( + math_ops.sqrt(math_ops.reduce_sum(array_ops.stack(dense_norms) * 2.0)) + if dense_norms else tf.constant(0.0)) grad_norm = math_ops.sqrt( - math_ops.reduce_sum(array_ops.stack(all_norms)) * 2.0) + math_ops.reduce_sum(array_ops.stack(all_norms)) * + 2.0) if all_norms else tf.constant(0.0) return sparse_norm, dense_norm, grad_norm @@ -496,14 +522,15 @@ def moving_average(name, value, decay): shape=value.get_shape(), dtype=value.dtype, initializer=init_ops.zeros_initializer(), - trainable=False) + trainable=False, + ) return moving_averages.assign_moving_average( moving_average_variable, value, decay, zero_debias=False) # quicker adaptation at the beginning if global_step is not None: n = math_ops.cast(global_step, dtypes.float32) - decay = math_ops.minimum(decay, n / (n + 1.)) + decay = math_ops.minimum(decay, n / (n + 1.0)) # update averages mean = moving_average('mean', log_norm, decay) @@ -515,13 +542,15 @@ def moving_average(name, value, decay): return max_norms, mean -def adaptive_clipping_fn(std_factor=2., - decay=0.95, - static_max_norm=None, - global_step=None, - report_summary=False, - epsilon=1e-8, - name=None): +def adaptive_clipping_fn( + std_factor=2.0, + decay=0.95, + static_max_norm=None, + global_step=None, + report_summary=False, + epsilon=1e-8, + name=None, +): """Adapt the clipping value using statistics on the norms. Implement adaptive gradient as presented in section 3.2.1 of @@ -605,8 +634,8 @@ def _multiply_gradients(grads_and_vars, gradient_multipliers): """Multiply specified gradients.""" multiplied_grads_and_vars = [] for grad, var in grads_and_vars: - if (grad is not None and - (var in gradient_multipliers or var.name in gradient_multipliers)): + if grad is not None and (var in gradient_multipliers or + var.name in gradient_multipliers): key = var if var in gradient_multipliers else var.name multiplier = gradient_multipliers[key] if isinstance(grad, indexed_slices.IndexedSlices): diff --git a/easy_rec/python/compat/queues.py b/easy_rec/python/compat/queues.py index c7063d966..a71a43bc2 100644 --- a/easy_rec/python/compat/queues.py +++ b/easy_rec/python/compat/queues.py @@ -28,7 +28,6 @@ from multiprocessing import context except ImportError: context = None - pass if context is not None: _ForkingPickler = context.reduction.ForkingPickler @@ -41,7 +40,6 @@ class Queue(object): - _sentinel = object() def __init__(self, ctx, maxsize=0, name=''): @@ -69,13 +67,32 @@ def __init__(self, ctx, maxsize=0, name=''): def __getstate__(self): context.assert_spawning(self) - return (self._ignore_epipe, self._maxsize, self._reader, self._writer, - self._rlock, self._wlock, self._sem, self._opid, self._name, - self._run) + return ( + self._ignore_epipe, + self._maxsize, + self._reader, + self._writer, + self._rlock, + self._wlock, + self._sem, + self._opid, + self._name, + self._run, + ) def __setstate__(self, state): - (self._ignore_epipe, self._maxsize, self._reader, self._writer, self._rlock, - self._wlock, self._sem, self._opid, self._name, self._run) = state + ( + self._ignore_epipe, + self._maxsize, + self._reader, + self._writer, + self._rlock, + self._wlock, + self._sem, + self._opid, + self._name, + self._run, + ) = state self._reset() def _after_fork(self): @@ -193,10 +210,19 @@ def _start_thread(self): self._buffer.clear() self._thread = threading.Thread( target=self._feed, - args=(self._buffer, self._notempty, self._send_bytes, self._wlock, - self._reader.close, self._writer.close, self._ignore_epipe, - self._on_queue_feeder_error, self._sem), - name='QueueFeederThread') + args=( + self._buffer, + self._notempty, + self._send_bytes, + self._wlock, + self._reader.close, + self._writer.close, + self._ignore_epipe, + self._on_queue_feeder_error, + self._sem, + ), + name='QueueFeederThread', + ) self._thread.daemon = True logging.debug('doing self._thread.start()') @@ -206,8 +232,10 @@ def _start_thread(self): if not self._joincancelled: self._jointhread = Finalize( self._thread, - Queue._finalize_join, [weakref.ref(self._thread)], - exitpriority=-5) + Queue._finalize_join, + [weakref.ref(self._thread)], + exitpriority=-5, + ) # Send sentinel to the thread queue object when garbage collected self._close = Finalize( @@ -232,8 +260,18 @@ def _finalize_close(buffer, notempty): buffer.append(Queue._sentinel) notempty.notify() - def _feed(self, buffer, notempty, send_bytes, writelock, reader_close, - writer_close, ignore_epipe, onerror, queue_sem): + def _feed( + self, + buffer, + notempty, + send_bytes, + writelock, + reader_close, + writer_close, + ignore_epipe, + onerror, + queue_sem, + ): logging.debug('starting thread to feed data to pipe') nacquire = notempty.acquire nrelease = notempty.release @@ -260,7 +298,6 @@ def _feed(self, buffer, notempty, send_bytes, writelock, reader_close, while self._run: obj = bpopleft() if obj is sentinel: - # logging.info('Queue[' + self._name + '] feeder thread got sentinel -- exiting: ' + str(self._run)) reader_close() writer_close() return @@ -308,4 +345,5 @@ def _on_queue_feeder_error(e, obj): For overriding by concurrent.futures. """ import traceback + traceback.print_exc() diff --git a/easy_rec/python/compat/regularizers.py b/easy_rec/python/compat/regularizers.py index 9609c037b..0ea11662f 100644 --- a/easy_rec/python/compat/regularizers.py +++ b/easy_rec/python/compat/regularizers.py @@ -30,8 +30,11 @@ from tensorflow.python.platform import tf_logging as logging __all__ = [ - 'l1_regularizer', 'l2_regularizer', 'l1_l2_regularizer', 'sum_regularizer', - 'apply_regularization' + 'l1_regularizer', + 'l2_regularizer', + 'l1_l2_regularizer', + 'sum_regularizer', + 'apply_regularization', ] @@ -53,10 +56,10 @@ def l1_regularizer(scale, scope=None): if isinstance(scale, numbers.Integral): raise ValueError('scale cannot be an integer: %s' % scale) if isinstance(scale, numbers.Real): - if scale < 0.: + if scale < 0.0: raise ValueError('Setting a scale less than 0 on a regularizer: %g' % scale) - if scale == 0.: + if scale == 0.0: logging.info('Scale of 0 disables regularizer.') return lambda _: None @@ -91,10 +94,10 @@ def l2_regularizer(scale, scope=None): if isinstance(scale, numbers.Integral): raise ValueError('scale cannot be an integer: %s' % (scale,)) if isinstance(scale, numbers.Real): - if scale < 0.: + if scale < 0.0: raise ValueError('Setting a scale less than 0 on a regularizer: %g.' % scale) - if scale == 0.: + if scale == 0.0: logging.info('Scale of 0 disables regularizer.') return lambda _: None @@ -128,9 +131,9 @@ def l1_l2_regularizer(scale_l1=1.0, scale_l2=1.0, scope=None): if isinstance(scale_l2, numbers.Integral): raise ValueError('scale_l2 cannot be an integer: %s' % (scale_l2,)) scope = scope or 'l1_l2_regularizer' - if scale_l1 == 0.: + if scale_l1 == 0.0: return l2_regularizer(scale_l2, scope) - if scale_l2 == 0.: + if scale_l2 == 0.0: return l1_regularizer(scale_l1, scope) return sum_regularizer([l1_regularizer(scale_l1), l2_regularizer(scale_l2)], diff --git a/easy_rec/python/compat/sok_optimizer.py b/easy_rec/python/compat/sok_optimizer.py index 7f368a9a1..a2e3ec361 100644 --- a/easy_rec/python/compat/sok_optimizer.py +++ b/easy_rec/python/compat/sok_optimizer.py @@ -18,14 +18,14 @@ from tensorflow.python.eager import context # from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -# from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradients -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops from easy_rec.python.compat.dynamic_variable import DynamicVariable +# from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import ( # NOQA + array_ops, gradients, resource_variable_ops, state_ops, +) + def OptimizerWrapper(optimizer): """Abbreviated as ``sok.experiment.OptimizerWrapper``. @@ -101,12 +101,14 @@ def __init__(self, optimizer): self._initial_vals[name] = slots[i] # self._optimizer._prepare() - def compute_gradients(self, - loss, - var_list=None, - aggregation_method=None, - colocate_gradients_with_ops=False, - grad_loss=None): + def compute_gradients( + self, + loss, + var_list=None, + aggregation_method=None, + colocate_gradients_with_ops=False, + grad_loss=None, + ): self._loss = loss tmp_grads = gradients.gradients(loss, var_list) return list(zip(tmp_grads, var_list)) @@ -140,7 +142,8 @@ def _create_slots_dynamic(self, var): dimension=var.dimension, initializer=self._initial_vals[slot_name], name='DynamicSlot', - trainable=False) + trainable=False, + ) else: tmp_config = var.config_dict # tmp_initializer = var.initializer_str @@ -151,7 +154,8 @@ def _create_slots_dynamic(self, var): var_type=var.backend_type, name='DynamicSlot', trainable=False, - **tmp_config) + **tmp_config, + ) self._optimizer._slots[slot_name][key] = slot @@ -199,7 +203,8 @@ def _dummy_finish(update_ops, name_scope): apply_updates = resource_variable_ops.assign_add_variable_op( global_step.handle, ops.convert_to_tensor(1, dtype=global_step.dtype), - name=name) + name=name, + ) else: apply_updates = state_ops.assign_add(global_step, 1, name=name) @@ -246,7 +251,8 @@ def apply_sparse_gradients(self, grads_and_vars, global_step=None, name=None): var_type=v.backend_type, name=tmp_slot_var_name, trainable=False, - **tmp_config) + **tmp_config, + ) self._optimizer._slots[slot_name][key] = slot else: @@ -334,7 +340,8 @@ def _create_slots_dynamic(self, var): var_type=var.backend_type, name='DynamicSlot', trainable=False, - **tmp_config) + **tmp_config, + ) self._optimizer._slots[key][slot_name] = slot def _var_key(self, var): @@ -387,7 +394,8 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): var_type=v.backend_type, name='DynamicSlot', trainable=False, - **tmp_config) + **tmp_config, + ) self._optimizer._slots[key][slot_name] = slot else: diff --git a/easy_rec/python/compat/sync_replicas_optimizer.py b/easy_rec/python/compat/sync_replicas_optimizer.py index 24c2921ba..d474165dc 100644 --- a/easy_rec/python/compat/sync_replicas_optimizer.py +++ b/easy_rec/python/compat/sync_replicas_optimizer.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================== """Synchronize replicas for training.""" + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -20,19 +21,17 @@ from tensorflow.core.framework import types_pb2 from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import optimizer -from tensorflow.python.training import queue_runner -from tensorflow.python.training import session_manager -from tensorflow.python.training import session_run_hook from tensorflow.python.util.tf_export import tf_export +from tensorflow.python.ops import ( # NOQA + array_ops, control_flow_ops, data_flow_ops, state_ops, variable_scope, + variables, +) +from tensorflow.python.training import ( # NOQA + optimizer, queue_runner, session_manager, session_run_hook, +) + # Please note that the gradients from replicas are averaged instead of summed # (as in the old sync_replicas_optimizer) so you need to increase the learning @@ -137,15 +136,17 @@ class SyncReplicasOptimizer(optimizer.Optimizer): sync_que_id = -1 - def __init__(self, - opt, - replicas_to_aggregate, - total_num_replicas=None, - variable_averages=None, - variables_to_average=None, - use_locking=False, - name='sync_replicas', - **extra_args): + def __init__( + self, + opt, + replicas_to_aggregate, + total_num_replicas=None, + variable_averages=None, + variables_to_average=None, + use_locking=False, + name='sync_replicas', + **extra_args, + ): """Construct a sync_replicas optimizer. Args: @@ -173,7 +174,9 @@ def __init__(self, super(SyncReplicasOptimizer, self).__init__(use_locking, name) logging.info( 'SyncReplicasV2: replicas_to_aggregate=%s; total_num_replicas=%s', - replicas_to_aggregate, total_num_replicas) + replicas_to_aggregate, + total_num_replicas, + ) self._opt = opt self._replicas_to_aggregate = replicas_to_aggregate self._gradients_applied = False @@ -256,7 +259,8 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): trainable=False, collections=[ops.GraphKeys.LOCAL_VARIABLES], dtype=global_step.dtype.base_dtype, - name='sync_rep_local_step') + name='sync_rep_local_step', + ) self.local_step_init_op = state_ops.assign(self._local_step, global_step) chief_init_ops = [self.local_step_init_op] @@ -275,7 +279,8 @@ def apply_gradients(self, grads_and_vars, global_step=None, name=None): grad_accum = data_flow_ops.ConditionalAccumulator( grad.dtype, shape=var.get_shape(), - shared_name=var.name + '/grad_accum') + shared_name=var.name + '/grad_accum', + ) train_ops.append( grad_accum.apply_grad(grad, local_step=self._local_step)) aggregated_grad.append( @@ -312,13 +317,13 @@ def _get_token_qname(): token_qname = _get_token_qname() logging.info('create sync_token_queue[%s]' % token_qname) with ops.device(global_step.device), ops.name_scope(''): - sync_token_queue = ( - data_flow_ops.FIFOQueue( - -1, - global_step.dtype.base_dtype, - shapes=(), - name=token_qname, - shared_name=token_qname)) + sync_token_queue = data_flow_ops.FIFOQueue( + -1, + global_step.dtype.base_dtype, + shapes=(), + name=token_qname, + shared_name=token_qname, + ) self._sync_token_queue = sync_token_queue self._is_sync_que_closed = sync_token_queue.is_closed() self._close_sync_que = sync_token_queue.close( @@ -327,13 +332,13 @@ def _get_token_qname(): # dummy_queue is passed to the queue runner. Don't use the real queues # because the queue runner doesn't automatically reopen it once it # closed queues in PS devices. - dummy_queue = ( - data_flow_ops.FIFOQueue( - 1, - types_pb2.DT_INT32, - shapes=(), - name='dummy_queue', - shared_name='dummy_queue')) + dummy_queue = data_flow_ops.FIFOQueue( + 1, + types_pb2.DT_INT32, + shapes=(), + name='dummy_queue', + shared_name='dummy_queue', + ) with ops.device(global_step.device), ops.name_scope(''): # Replicas have to wait until they can get a token from the token queue. @@ -490,21 +495,24 @@ def begin(self): 'the hook.') if self._is_chief: self._local_init_op = self._sync_optimizer.chief_init_op - self._ready_for_local_init_op = ( - self._sync_optimizer.ready_for_local_init_op) + self._ready_for_local_init_op = self._sync_optimizer.ready_for_local_init_op self._init_tokens_op = self._sync_optimizer.get_init_tokens_op( self._num_tokens) else: self._local_init_op = self._sync_optimizer.local_step_init_op - self._ready_for_local_init_op = ( - self._sync_optimizer.ready_for_local_init_op) + self._ready_for_local_init_op = self._sync_optimizer.ready_for_local_init_op self._init_tokens_op = None def after_create_session(self, session, coord): """Runs SyncReplicasOptimizer initialization ops.""" - local_init_success, msg = session_manager._ready( # pylint: disable=protected-access - self._ready_for_local_init_op, session, - 'Model is not ready for SyncReplicasOptimizer local init.') + ( + local_init_success, + msg, + ) = session_manager._ready( # pylint: disable=protected-access + self._ready_for_local_init_op, + session, + 'Model is not ready for SyncReplicasOptimizer local init.', + ) if not local_init_success: raise RuntimeError( 'Init operations did not make model ready for SyncReplicasOptimizer ' diff --git a/easy_rec/python/compat/weight_decay_optimizers.py b/easy_rec/python/compat/weight_decay_optimizers.py index 47a755e0f..7cff692eb 100755 --- a/easy_rec/python/compat/weight_decay_optimizers.py +++ b/easy_rec/python/compat/weight_decay_optimizers.py @@ -13,20 +13,21 @@ # limitations under the License. # ============================================================================== """Base class to make optimizers weight decay ready.""" + from __future__ import absolute_import from __future__ import division from __future__ import print_function from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import state_ops from tensorflow.python.training import adam from tensorflow.python.training import momentum as momentum_opt from tensorflow.python.training import optimizer from tensorflow.python.util.tf_export import tf_export +from tensorflow.python.ops import ( # NOQA + array_ops, control_flow_ops, resource_variable_ops, state_ops, +) + class DecoupledWeightDecayExtension(object): """This class allows to extend optimizers with decoupled weight decay. @@ -91,16 +92,18 @@ def __init__(self, weight_decay, **kwargs): self._weight_decay_tensor = None super(DecoupledWeightDecayExtension, self).__init__(**kwargs) - def minimize(self, - loss, - global_step=None, - var_list=None, - gate_gradients=optimizer.Optimizer.GATE_OP, - aggregation_method=None, - colocate_gradients_with_ops=False, - name=None, - grad_loss=None, - decay_var_list=None): + def minimize( + self, + loss, + global_step=None, + var_list=None, + gate_gradients=optimizer.Optimizer.GATE_OP, + aggregation_method=None, + colocate_gradients_with_ops=False, + name=None, + grad_loss=None, + decay_var_list=None, + ): """Add operations to minimize `loss` by updating `var_list` with decay. This function is the same as Optimizer.minimize except that it allows to @@ -139,7 +142,8 @@ def minimize(self, aggregation_method=aggregation_method, colocate_gradients_with_ops=colocate_gradients_with_ops, name=name, - grad_loss=grad_loss) + grad_loss=grad_loss, + ) def apply_gradients(self, grads_and_vars, @@ -311,13 +315,15 @@ class MomentumWOptimizer(DecoupledWeightDecayExtension, ``` """ - def __init__(self, - weight_decay, - learning_rate, - momentum, - use_locking=False, - name='MomentumW', - use_nesterov=False): + def __init__( + self, + weight_decay, + learning_rate, + momentum, + use_locking=False, + name='MomentumW', + use_nesterov=False, + ): """Construct a new MomentumW optimizer. For further information see the documentation of the Momentum Optimizer. @@ -347,7 +353,8 @@ def __init__(self, momentum=momentum, use_locking=use_locking, name=name, - use_nesterov=use_nesterov) + use_nesterov=use_nesterov, + ) @tf_export('contrib.opt.AdamWOptimizer') @@ -374,14 +381,16 @@ class AdamWOptimizer(DecoupledWeightDecayExtension, adam.AdamOptimizer): ``` """ - def __init__(self, - weight_decay, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - use_locking=False, - name='AdamW'): + def __init__( + self, + weight_decay, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + use_locking=False, + name='AdamW', + ): """Construct a new AdamW optimizer. For further information see the documentation of the Adam Optimizer. @@ -407,7 +416,8 @@ def __init__(self, beta2=beta2, epsilon=epsilon, use_locking=use_locking, - name=name) + name=name, + ) try: @@ -437,14 +447,16 @@ class AdamAsyncWOptimizer(DecoupledWeightDecayExtension, AdamAsyncOptimizer): ``` """ - def __init__(self, - weight_decay, - learning_rate=0.001, - beta1=0.9, - beta2=0.999, - epsilon=1e-8, - use_locking=False, - name='AdamAsyncW'): + def __init__( + self, + weight_decay, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + use_locking=False, + name='AdamAsyncW', + ): """Construct a new AdamW optimizer. For further information see the documentation of the Adam Optimizer. @@ -470,6 +482,8 @@ def __init__(self, beta2=beta2, epsilon=epsilon, use_locking=use_locking, - name=name) + name=name, + ) + except ImportError: pass diff --git a/easy_rec/python/core/easyrec_metrics/__init__.py b/easy_rec/python/core/easyrec_metrics/__init__.py index cba3ebc08..f5d82c865 100644 --- a/easy_rec/python/core/easyrec_metrics/__init__.py +++ b/easy_rec/python/core/easyrec_metrics/__init__.py @@ -13,10 +13,10 @@ if distribute_eval == 'True': if pai_util.is_on_pai() or tf.__version__ <= '1.13': logging.info('Will use distribute pai_tf metrics impl') - from easy_rec.python.core.easyrec_metrics import distribute_metrics_impl_pai as metrics_tf + from easy_rec.python.core.easyrec_metrics import distribute_metrics_impl_pai as metrics_tf # NOQA else: logging.info('Will use distribute tf metrics impl') - from easy_rec.python.core.easyrec_metrics import distribute_metrics_impl_tf as metrics_tf + from easy_rec.python.core.easyrec_metrics import distribute_metrics_impl_tf as metrics_tf # NOQA else: if tf.__version__ >= '2.0': from tensorflow.compat.v1 import metrics as metrics_tf diff --git a/easy_rec/python/core/learning_schedules.py b/easy_rec/python/core/learning_schedules.py index fcd1446e8..506aea3af 100644 --- a/easy_rec/python/core/learning_schedules.py +++ b/easy_rec/python/core/learning_schedules.py @@ -22,14 +22,16 @@ tf = tf.compat.v1 -def exponential_decay_with_burnin(global_step, - learning_rate_base, - learning_rate_decay_steps, - learning_rate_decay_factor, - burnin_learning_rate=0.0, - burnin_steps=0, - min_learning_rate=0.0, - staircase=True): +def exponential_decay_with_burnin( + global_step, + learning_rate_base, + learning_rate_decay_steps, + learning_rate_decay_factor, + burnin_learning_rate=0.0, + burnin_steps=0, + min_learning_rate=0.0, + staircase=True, +): """Exponential decay schedule with burn-in period. In this schedule, learning rate is fixed at burnin_learning_rate @@ -64,21 +66,27 @@ def exponential_decay_with_burnin(global_step, global_step - burnin_steps, learning_rate_decay_steps, learning_rate_decay_factor, - staircase=staircase) + staircase=staircase, + ) return tf.maximum( tf.where( tf.less(tf.cast(global_step, tf.int32), tf.constant(burnin_steps)), - burnin_rate, post_burnin_learning_rate), + burnin_rate, + post_burnin_learning_rate, + ), min_learning_rate, - name='learning_rate') - - -def cosine_decay_with_warmup(global_step, - learning_rate_base, - total_steps, - warmup_learning_rate=0.0, - warmup_steps=0, - hold_base_rate_steps=0): + name='learning_rate', + ) + + +def cosine_decay_with_warmup( + global_step, + learning_rate_base, + total_steps, + warmup_learning_rate=0.0, + warmup_steps=0, + hold_base_rate_steps=0, +): """Cosine decay schedule with warm up period. Cosine annealing learning rate as described in: @@ -109,13 +117,16 @@ def cosine_decay_with_warmup(global_step, 'or equal to warmup_learning_rate.') if total_steps < warmup_steps: raise ValueError('total_steps must be larger or equal to ' 'warmup_steps.') - learning_rate = 0.5 * learning_rate_base * (1 + tf.cos( + learning_rate = (0.5 * learning_rate_base * (1 + tf.cos( np.pi * (tf.cast(global_step, tf.float32) - warmup_steps - hold_base_rate_steps) / - float(total_steps - warmup_steps - hold_base_rate_steps))) + float(total_steps - warmup_steps - hold_base_rate_steps)))) if hold_base_rate_steps > 0: - learning_rate = tf.where(global_step > warmup_steps + hold_base_rate_steps, - learning_rate, learning_rate_base) + learning_rate = tf.where( + global_step > warmup_steps + hold_base_rate_steps, + learning_rate, + learning_rate_base, + ) if warmup_steps > 0: slope = (learning_rate_base - warmup_learning_rate) / warmup_steps warmup_rate = slope * tf.cast(global_step, @@ -180,20 +191,24 @@ def manual_stepping(global_step, boundaries, rates, warmup=False): rate_index = tf.reduce_max( tf.where( tf.greater_equal(global_step, boundaries), - list(range(num_boundaries)), [0] * num_boundaries)) + list(range(num_boundaries)), + [0] * num_boundaries, + )) return tf.reduce_sum( rates * tf.one_hot(rate_index, depth=num_boundaries), name='learning_rate') -def transformer_policy(global_step, - learning_rate, - d_model, - warmup_steps, - step_scaling_rate=1.0, - max_lr=None, - coefficient=1.0, - dtype=tf.float32): +def transformer_policy( + global_step, + learning_rate, + d_model, + warmup_steps, + step_scaling_rate=1.0, + max_lr=None, + coefficient=1.0, + dtype=tf.float32, +): """Transformer's learning rate schedule. Transformer's learning rate policy from diff --git a/easy_rec/python/core/metrics.py b/easy_rec/python/core/metrics.py index bd7cb0976..6209e807e 100644 --- a/easy_rec/python/core/metrics.py +++ b/easy_rec/python/core/metrics.py @@ -8,16 +8,16 @@ import numpy as np import tensorflow as tf from sklearn import metrics as sklearn_metrics -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variable_scope from easy_rec.python.utils.estimator_utils import get_task_index_and_num -from easy_rec.python.utils.io_util import read_data_from_json_path -from easy_rec.python.utils.io_util import save_data_to_json_path from easy_rec.python.utils.shape_utils import get_shape_list +from tensorflow.python.ops import array_ops, math_ops, state_ops, variable_scope # NOQA + +from easy_rec.python.utils.io_util import ( # NOQA + read_data_from_json_path, save_data_to_json_path, +) + if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -30,6 +30,7 @@ def max_f1(label, predictions): predictions: Estimated targets as returned by a model. """ from easy_rec.python.core.easyrec_metrics import metrics_tf + num_thresholds = 200 kepsilon = 1e-7 thresholds = [ @@ -41,7 +42,7 @@ def max_f1(label, predictions): precision_update_ops = [] recall_update_ops = [] for threshold in thresholds: - pred = (predictions > threshold) + pred = predictions > threshold precision, precision_update_op = metrics_tf.precision( labels=label, predictions=pred, name='precision_%s' % threshold) recall, recall_update_op = metrics_tf.recall( @@ -70,8 +71,11 @@ def _separated_auc_impl(labels, predictions, keys, reduction='mean'): * "mean_by_sample_num": weighted mean with sample num of different keys * "mean_by_positive_num": weighted mean with positive sample num of different keys """ - assert reduction in ['mean', 'mean_by_sample_num', 'mean_by_positive_num'], \ - 'reduction method must in mean | mean_by_sample_num | mean_by_positive_num' + assert reduction in [ + 'mean', + 'mean_by_sample_num', + 'mean_by_positive_num', + ], 'reduction method must in mean | mean_by_sample_num | mean_by_positive_num' separated_label = defaultdict(list) separated_prediction = defaultdict(list) separated_weights = defaultdict(int) @@ -132,21 +136,25 @@ def value_pyfunc(pos_neg_arr, total_pos_neg): trainable=False, collections=[tf.GraphKeys.METRIC_VARIABLES], initializer=tf.zeros_initializer(), - dtype=tf.int64) + dtype=tf.int64, + ) total_var = variable_scope.get_variable( name='total_cnt', shape=[2], trainable=False, collections=[tf.GraphKeys.METRIC_VARIABLES], initializer=tf.zeros_initializer(), - dtype=tf.int64) + dtype=tf.int64, + ) pred_bins = math_ops.cast(predictions * num_thresholds, dtype=tf.int32) labels = math_ops.cast(labels, dtype=tf.int32) labels = array_ops.reshape(labels, [-1, 1]) pred_bins = array_ops.reshape(pred_bins, [-1, 1]) update_op0 = state_ops.scatter_nd_add( - neg_pos_var, tf.concat([labels, pred_bins], axis=1), - array_ops.ones(tf.shape(labels)[0], dtype=tf.int64)) + neg_pos_var, + tf.concat([labels, pred_bins], axis=1), + array_ops.ones(tf.shape(labels)[0], dtype=tf.int64), + ) total_pos = math_ops.reduce_sum(labels) total_neg = array_ops.shape(labels)[0] - total_pos total_add = math_ops.cast(tf.stack([total_neg, total_pos]), dtype=tf.int64) @@ -174,8 +182,11 @@ def _distribute_separated_auc_impl(labels, * "mean_by_sample_num": weighted mean with sample num of different keys * "mean_by_positive_num": weighted mean with positive sample num of different keys """ - assert reduction in ['mean', 'mean_by_sample_num', 'mean_by_positive_num'], \ - 'reduction method must in mean | mean_by_sample_num | mean_by_positive_num' + assert reduction in [ + 'mean', + 'mean_by_sample_num', + 'mean_by_positive_num', + ], 'reduction method must in mean | mean_by_sample_num | mean_by_positive_num' separated_label = defaultdict(list) separated_prediction = defaultdict(list) separated_weights = defaultdict(int) @@ -200,7 +211,8 @@ def update_pyfunc(labels, predictions, keys): separated_weights[key] += label.item() for name, data in zip( ['separated_label', 'separated_prediction', 'separated_weights'], - [separated_label, separated_prediction, separated_weights]): + [separated_label, separated_prediction, separated_weights], + ): cur_json_name = metric_name + '__' + cur_work_device + '__' + name + '.json' cur_json_path = os.path.join(eval_tmp_results_dir, cur_json_name) save_data_to_json_path(cur_json_path, data) @@ -209,24 +221,26 @@ def value_pyfunc(): for task_i in range(1, task_num): work_device_i = 'job_worker__task_' + str(task_i) for name in [ - 'separated_label', 'separated_prediction', 'separated_weights' + 'separated_label', + 'separated_prediction', + 'separated_weights', ]: json_name_i = metric_name + '__' + work_device_i + '__' + name + '.json' json_path_i = os.path.join(eval_tmp_results_dir, json_name_i) data_i = read_data_from_json_path(json_path_i) - if (name == 'separated_label'): + if name == 'separated_label': separated_label.update({ key: separated_label.get(key, []) + data_i.get(key, []) for key in set( list(separated_label.keys()) + list(data_i.keys())) }) - elif (name == 'separated_prediction'): + elif name == 'separated_prediction': separated_prediction.update({ key: separated_prediction.get(key, []) + data_i.get(key, []) for key in set( list(separated_prediction.keys()) + list(data_i.keys())) }) - elif (name == 'separated_weights'): + elif name == 'separated_weights': if reduction == 'mean': separated_weights.update(data_i) else: @@ -312,6 +326,7 @@ def metric_learning_recall_at_k(k, embed_normed: indicator of whether the input embeddings are l2_normalized """ from easy_rec.python.core.easyrec_metrics import metrics_tf + # make sure embedding should be l2-normalized if not embed_normed: embeddings = tf.nn.l2_normalize(embeddings, axis=1) @@ -353,6 +368,7 @@ def metric_learning_average_precision_at_k(k, session_ids=None, embed_normed=False): from easy_rec.python.core.easyrec_metrics import metrics_tf + # make sure embedding should be l2-normalized if not embed_normed: embeddings = tf.nn.l2_normalize(embeddings, axis=1) diff --git a/easy_rec/python/eval.py b/easy_rec/python/eval.py index d41c3d7ae..bb4205764 100644 --- a/easy_rec/python/eval.py +++ b/easy_rec/python/eval.py @@ -15,24 +15,33 @@ from easy_rec.python.utils import ds_util from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils.distribution_utils import set_tf_config_and_get_distribute_eval_worker_num_on_ds # NOQA +from easy_rec.python.utils.distribution_utils import ( # NOQA + set_tf_config_and_get_distribute_eval_worker_num_on_ds,) + if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + level=logging.INFO, +) tf.app.flags.DEFINE_string('pipeline_config_path', None, 'Path to pipeline config ' 'file.') tf.app.flags.DEFINE_string( - 'checkpoint_path', None, 'checkpoint to be evaled ' + 'checkpoint_path', + None, + 'checkpoint to be evaled ' ' if not specified, use the latest checkpoint in ' - 'train_config.model_dir') + 'train_config.model_dir', +) tf.app.flags.DEFINE_multi_string( - 'eval_input_path', None, 'eval data path, if specified will ' - 'override pipeline_config.eval_input_path') + 'eval_input_path', + None, + 'eval data path, if specified will ' + 'override pipeline_config.eval_input_path', +) tf.app.flags.DEFINE_string('model_dir', None, help='will update the model_dir') tf.app.flags.DEFINE_string('odps_config', None, help='odps config path') tf.app.flags.DEFINE_string('eval_result_path', 'eval_result.txt', @@ -73,20 +82,27 @@ def main(argv): estimator_utils.init_hvd() elif pipeline_config.train_config.train_distribute in [ DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy + DistributionStrategy.SokStrategy, ]: estimator_utils.init_hvd() estimator_utils.init_sok() if FLAGS.distribute_eval: os.environ['distribute_eval'] = 'True' - eval_result = distribute_evaluate(pipeline_config, FLAGS.checkpoint_path, - FLAGS.eval_input_path, - FLAGS.eval_result_path) + eval_result = distribute_evaluate( + pipeline_config, + FLAGS.checkpoint_path, + FLAGS.eval_input_path, + FLAGS.eval_result_path, + ) else: os.environ['distribute_eval'] = 'False' - eval_result = evaluate(pipeline_config, FLAGS.checkpoint_path, - FLAGS.eval_input_path, FLAGS.eval_result_path) + eval_result = evaluate( + pipeline_config, + FLAGS.checkpoint_path, + FLAGS.eval_input_path, + FLAGS.eval_result_path, + ) if eval_result is not None: # when distribute evaluate, only master has eval_result. for key in sorted(eval_result): diff --git a/easy_rec/python/export.py b/easy_rec/python/export.py index c1a8ce670..ac59d50dc 100644 --- a/easy_rec/python/export.py +++ b/easy_rec/python/export.py @@ -21,7 +21,8 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + level=logging.INFO, +) tf.app.flags.DEFINE_string('pipeline_config_path', None, 'Path to pipeline config ' 'file.') @@ -125,7 +126,7 @@ def main(argv): estimator_utils.init_hvd() elif pipeline_config.train_config.train_distribute in [ DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy + DistributionStrategy.SokStrategy, ]: estimator_utils.init_hvd() estimator_utils.init_sok() @@ -135,9 +136,14 @@ def main(argv): if gfile.IsDirectory(FLAGS.export_dir): gfile.DeleteRecursively(FLAGS.export_dir) - export_out_dir = export(FLAGS.export_dir, pipeline_config_path, - FLAGS.checkpoint_path, FLAGS.asset_files, - FLAGS.verbose, **extra_params) + export_out_dir = export( + FLAGS.export_dir, + pipeline_config_path, + FLAGS.checkpoint_path, + FLAGS.asset_files, + FLAGS.verbose, + **extra_params, + ) if FLAGS.export_done_file: flag_file = os.path.join(export_out_dir, FLAGS.export_done_file) diff --git a/easy_rec/python/feature_column/feature_column.py b/easy_rec/python/feature_column/feature_column.py index 8701b55fc..53791c727 100644 --- a/easy_rec/python/feature_column/feature_column.py +++ b/easy_rec/python/feature_column/feature_column.py @@ -33,9 +33,13 @@ def __init__(self, embedding_name, index, sequence_combiner=None): self.sequence_combiner = sequence_combiner -EVParams = collections.namedtuple('EVParams', [ - 'filter_freq', 'steps_to_live', 'use_cache', 'init_capacity', 'max_capacity' -]) +EVParams = collections.namedtuple( + 'EVParams', + [ + 'filter_freq', 'steps_to_live', 'use_cache', 'init_capacity', + 'max_capacity' + ], +) class FeatureColumnParser(object): @@ -75,9 +79,10 @@ def __init__(self, self._global_ev_params = self._build_ev_params(ev_params) def _cmp_embed_config(a, b): - return a.embedding_dim == b.embedding_dim and a.combiner == b.combiner and\ - a.initializer == b.initializer and a.max_partitions == b.max_partitions and\ - a.embedding_name == b.embedding_name + return (a.embedding_dim == b.embedding_dim and + a.combiner == b.combiner and a.initializer == b.initializer and + a.max_partitions == b.max_partitions and + a.embedding_name == b.embedding_name) for config in self._feature_configs: if not config.HasField('embedding_name'): @@ -85,9 +90,12 @@ def _cmp_embed_config(a, b): embed_name = config.embedding_name if embed_name in self._share_embed_names: - assert _cmp_embed_config(config, self._share_embed_infos[embed_name]),\ + assert _cmp_embed_config(config, self._share_embed_infos[embed_name]), ( 'shared embed info of [%s] is not matched [%s] vs [%s]' % ( - embed_name, config, self._share_embed_infos[embed_name]) + embed_name, + config, + self._share_embed_infos[embed_name], + )) self._share_embed_names[embed_name] += 1 if config.feature_type == FeatureConfig.FeatureType.SequenceFeature: self._share_embed_infos[embed_name] = copy_obj(config) @@ -105,9 +113,11 @@ def _cmp_embed_config(a, b): logging.info('shared embeddings[num=%d]' % len(self._share_embed_names)) for embed_name in self._share_embed_names: - logging.info('\t%s: share_num[%d], share_info[%s]' % - (embed_name, self._share_embed_names[embed_name], - self._share_embed_infos[embed_name])) + logging.info('\t%s: share_num[%d], share_info[%s]' % ( + embed_name, + self._share_embed_names[embed_name], + self._share_embed_infos[embed_name], + )) self._deep_share_embed_columns = { embed_name: [] for embed_name in self._share_embed_names } @@ -161,7 +171,8 @@ def _cmp_embed_config(a, b): shared_embedding_collection_name=embed_name, combiner=self._share_embed_infos[embed_name].combiner, partitioner=partitioner, - ev_params=ev_params) + ev_params=ev_params, + ) config = self._share_embed_infos[embed_name] max_seq_len = config.max_seq_len if config.HasField( 'max_seq_len') else -1 @@ -178,7 +189,8 @@ def _cmp_embed_config(a, b): shared_embedding_collection_name=embed_name + '_wide', combiner='sum', partitioner=partitioner, - ev_params=ev_params) + ev_params=ev_params, + ) config = self._share_embed_infos[embed_name] max_seq_len = config.max_seq_len if config.HasField( 'max_seq_len') else -1 @@ -222,7 +234,8 @@ def is_wide(self, config): if feature_name not in self._wide_deep_dict: raise FeatureKeyError(feature_name) return self._wide_deep_dict[feature_name] in [ - WideOrDeep.WIDE, WideOrDeep.WIDE_AND_DEEP + WideOrDeep.WIDE, + WideOrDeep.WIDE_AND_DEEP, ] def is_deep(self, config): @@ -234,7 +247,8 @@ def is_deep(self, config): if feature_name not in self._wide_deep_dict: raise FeatureKeyError(feature_name) return self._wide_deep_dict[feature_name] in [ - WideOrDeep.DEEP, WideOrDeep.WIDE_AND_DEEP + WideOrDeep.DEEP, + WideOrDeep.WIDE_AND_DEEP, ] def get_feature_vocab_size(self, feature): @@ -266,27 +280,30 @@ def parse_id_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] hash_bucket_size = self._get_hash_bucket_size(config) if hash_bucket_size > 0: fc = feature_column.categorical_column_with_hash_bucket( feature_name, hash_bucket_size=hash_bucket_size, - feature_name=feature_name) + feature_name=feature_name, + ) elif config.vocab_list: fc = feature_column.categorical_column_with_vocabulary_list( feature_name, default_value=0, vocabulary_list=config.vocab_list, - feature_name=feature_name) + feature_name=feature_name, + ) elif config.vocab_file: fc = feature_column.categorical_column_with_vocabulary_file( feature_name, default_value=0, vocabulary_file=config.vocab_file, vocabulary_size=self._get_vocab_size(config.vocab_file), - feature_name=feature_name) + feature_name=feature_name, + ) else: use_ev = self._global_ev_params or config.HasField('ev_params') num_buckets = sys.maxsize if use_ev else config.num_buckets @@ -308,28 +325,31 @@ def parse_tag_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] hash_bucket_size = self._get_hash_bucket_size(config) if hash_bucket_size > 0: tag_fc = feature_column.categorical_column_with_hash_bucket( feature_name, hash_bucket_size, dtype=tf.string, - feature_name=feature_name) + feature_name=feature_name, + ) elif config.vocab_list: tag_fc = feature_column.categorical_column_with_vocabulary_list( feature_name, default_value=0, vocabulary_list=config.vocab_list, - feature_name=feature_name) + feature_name=feature_name, + ) elif config.vocab_file: tag_fc = feature_column.categorical_column_with_vocabulary_file( feature_name, default_value=0, vocabulary_file=config.vocab_file, vocabulary_size=self._get_vocab_size(config.vocab_file), - feature_name=feature_name) + feature_name=feature_name, + ) else: use_ev = self._global_ev_params or config.HasField('ev_params') num_buckets = sys.maxsize if use_ev else config.num_buckets @@ -356,8 +376,8 @@ def parse_raw_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] fc = feature_column.numeric_column( key=feature_name, shape=(config.raw_input_dim,), @@ -391,11 +411,13 @@ def parse_raw_feature(self, config): feature_name + '_raw_proj_id', config.raw_input_dim, default_value=0, - feature_name=feature_name) + feature_name=feature_name, + ) wgt_fc = feature_column.weighted_categorical_column( tmp_id_col, weight_feature_key=feature_name + '_raw_proj_val', - dtype=tf.float32) + dtype=tf.float32, + ) if self.is_wide(config): self._add_wide_embedding_column(wgt_fc, config) if self.is_deep(config): @@ -412,8 +434,8 @@ def parse_expr_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] fc = feature_column.numeric_column( feature_name, shape=(1,), feature_name=feature_name) if self.is_wide(config): @@ -427,8 +449,8 @@ def parse_combo_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else None + feature_name = config.feature_name if config.HasField( + 'feature_name') else None assert len(config.input_names) >= 2 if len(config.combo_join_sep) == 0: @@ -442,12 +464,14 @@ def parse_combo_feature(self, config): input_names, self._get_hash_bucket_size(config), hash_key=None, - feature_name=feature_name) + feature_name=feature_name, + ) else: fc = feature_column.categorical_column_with_hash_bucket( feature_name, hash_bucket_size=self._get_hash_bucket_size(config), - feature_name=feature_name) + feature_name=feature_name, + ) if self.is_wide(config): self._add_wide_embedding_column(fc, config) @@ -460,8 +484,8 @@ def parse_lookup_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] assert config.HasField('hash_bucket_size') hash_bucket_size = self._get_hash_bucket_size(config) fc = feature_column.categorical_column_with_hash_bucket( @@ -481,11 +505,13 @@ def parse_sequence_feature(self, config): Args: config: instance of easy_rec.python.protos.feature_config_pb2.FeatureConfig """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] sub_feature_type = config.sub_feature_type - assert sub_feature_type in [config.IdFeature, config.RawFeature], \ - 'Current sub_feature_type only support IdFeature and RawFeature.' + assert sub_feature_type in [ + config.IdFeature, + config.RawFeature, + ], 'Current sub_feature_type only support IdFeature and RawFeature.' if sub_feature_type == config.IdFeature: if config.HasField('hash_bucket_size'): hash_bucket_size = self._get_hash_bucket_size(config) @@ -493,20 +519,23 @@ def parse_sequence_feature(self, config): feature_name, hash_bucket_size, dtype=tf.string, - feature_name=feature_name) + feature_name=feature_name, + ) elif config.vocab_list: fc = sequence_feature_column.sequence_categorical_column_with_vocabulary_list( feature_name, default_value=0, vocabulary_list=config.vocab_list, - feature_name=feature_name) + feature_name=feature_name, + ) elif config.vocab_file: fc = sequence_feature_column.sequence_categorical_column_with_vocabulary_file( feature_name, default_value=0, vocabulary_file=config.vocab_file, vocabulary_size=self._get_vocab_size(config.vocab_file), - feature_name=feature_name) + feature_name=feature_name, + ) else: use_ev = self._global_ev_params or config.HasField('ev_params') num_buckets = sys.maxsize if use_ev else config.num_buckets @@ -514,15 +543,17 @@ def parse_sequence_feature(self, config): feature_name, num_buckets, default_value=0, - feature_name=feature_name) + feature_name=feature_name, + ) else: # raw feature bounds = None fc = sequence_feature_column.sequence_numeric_column( feature_name, shape=(1,), feature_name=feature_name) if config.hash_bucket_size > 0: hash_bucket_size = self._get_hash_bucket_size(config) - assert sub_feature_type == config.IdFeature, \ - 'You should set sub_feature_type to IdFeature to use hash_bucket_size.' + assert ( + sub_feature_type == config.IdFeature + ), 'You should set sub_feature_type to IdFeature to use hash_bucket_size.' elif config.boundaries: bounds = list(config.boundaries) bounds.sort() @@ -548,11 +579,13 @@ def parse_sequence_feature(self, config): feature_name + '_raw_proj_id', config.raw_input_dim, default_value=0, - feature_name=feature_name) + feature_name=feature_name, + ) wgt_fc = sequence_feature_column.sequence_weighted_categorical_column( tmp_id_col, weight_feature_key=feature_name + '_raw_proj_val', - dtype=tf.float32) + dtype=tf.float32, + ) fc = wgt_fc else: fc = sequence_feature_column.sequence_numeric_column_with_raw_column( @@ -599,8 +632,8 @@ def _add_wide_embedding_column(self, fc, config): We use embedding to simulate wide column, which is more efficient than indicator column for sparse features """ - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] assert self._wide_output_dim > 0, 'wide_output_dim is not set' if config.embedding_name in self._wide_share_embed_columns: wide_fc = self._add_shared_embedding_column( @@ -619,13 +652,14 @@ def _add_wide_embedding_column(self, fc, config): combiner='sum', initializer=initializer, partitioner=self._build_partitioner(config), - ev_params=ev_params) + ev_params=ev_params, + ) self._wide_columns[feature_name] = wide_fc def _add_deep_embedding_column(self, fc, config): """Generate deep feature columns.""" - feature_name = config.feature_name if config.HasField('feature_name') \ - else config.input_names[0] + feature_name = config.feature_name if config.HasField( + 'feature_name') else config.input_names[0] assert config.embedding_dim > 0, 'embedding_dim is not set for %s' % feature_name self._feature_vocab_size[feature_name] = fc.num_buckets if config.embedding_name in self._deep_share_embed_columns: @@ -644,7 +678,8 @@ def _add_deep_embedding_column(self, fc, config): combiner=config.combiner, initializer=initializer, partitioner=self._build_partitioner(config), - ev_params=ev_params) + ev_params=ev_params, + ) fc.max_seq_length = config.max_seq_len if config.HasField( 'max_seq_len') else -1 @@ -660,5 +695,8 @@ def _build_ev_params(self, ev_params): ev_params = EVParams( ev_params.filter_freq, ev_params.steps_to_live if ev_params.steps_to_live > 0 else None, - ev_params.use_cache, ev_params.init_capacity, ev_params.max_capacity) + ev_params.use_cache, + ev_params.init_capacity, + ev_params.max_capacity, + ) return ev_params diff --git a/easy_rec/python/feature_column/feature_group.py b/easy_rec/python/feature_column/feature_group.py index b04a635ad..b4dcc83da 100644 --- a/easy_rec/python/feature_column/feature_group.py +++ b/easy_rec/python/feature_column/feature_group.py @@ -2,8 +2,9 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import re -from easy_rec.python.protos.feature_config_pb2 import FeatureGroupConfig -from easy_rec.python.protos.feature_config_pb2 import WideOrDeep +from easy_rec.python.protos.feature_config_pb2 import ( # NOQA + FeatureGroupConfig, WideOrDeep, +) class FeatureGroup(object): diff --git a/easy_rec/python/hpo/generate_hpo_sql.py b/easy_rec/python/hpo/generate_hpo_sql.py index 648be7c38..6b7dc68ea 100644 --- a/easy_rec/python/hpo/generate_hpo_sql.py +++ b/easy_rec/python/hpo/generate_hpo_sql.py @@ -20,7 +20,8 @@ '--cluster', type=str, help='specify tensorflow train jobs cluster parameter', - default=None) + default=None, + ) parser.add_argument('--bucket', type=str, help='oss bucket', default=None) parser.add_argument( '--hpo_param_path', type=str, help='hpo param path', default=None) @@ -36,7 +37,8 @@ '--algo_proj_name', type=str, help='algorithm project name', - default='algo_public') + default='algo_public', + ) parser.add_argument( '--algo_res_proj', type=str, help='algo resource project', default=None) parser.add_argument( @@ -59,7 +61,7 @@ else: fout.write(' -Dtrain_tables=%s\n' % args.train_tables) fout.write(' -Deval_tables=%s\n' % args.eval_tables) - fout.write(' -Dcluster=\'%s\'\n' % args.cluster) + fout.write(" -Dcluster='%s'\n" % args.cluster) fout.write(' -Darn=%s\n' % args.role_arn) fout.write(' -Dbuckets=%s\n' % args.bucket) fout.write(' -Dhpo_param_path=%s\n' % args.hpo_param_path) diff --git a/easy_rec/python/hpo/pai_hpo.py b/easy_rec/python/hpo/pai_hpo.py index 8d6edaa19..3473552d7 100644 --- a/easy_rec/python/hpo/pai_hpo.py +++ b/easy_rec/python/hpo/pai_hpo.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Hyperparameter search demo for easy_rec on pai.""" + import json import logging import os @@ -66,13 +67,26 @@ def get_tuner(data, max_parallel, max_trial_num): max_parallel=max_parallel, max_trial_num=max_trial_num, mode='local', - user_id='your_cloud_id') + user_id='your_cloud_id', + ) return tuner -def hpo_config(config_path, hyperparams, environment, exp_dir, tables, - train_tables, eval_tables, cluster, algo_proj_name, - algo_res_proj, algo_version, metric_name, odps_config_path): +def hpo_config( + config_path, + hyperparams, + environment, + exp_dir, + tables, + train_tables, + eval_tables, + cluster, + algo_proj_name, + algo_res_proj, + algo_version, + metric_name, + odps_config_path, +): earlystop = {'type': 'large_is_better', 'max_runtime': 3600 * 12} algorithm = { 'type': 'gp', @@ -118,14 +132,29 @@ def _add_prefix(table_name): sql_path = '%s/train_ext_hpo_{{ trial.id }}.sql' % tmp_dir cmd_args = [ - 'python', '-m', 'easy_rec.python.hpo.generate_hpo_sql', '--sql_path', - sql_path, '--config_path', config_path, '--cluster', cluster, '--bucket', - bucket, '--hpo_param_path', - os.path.join(bucket, param_path), '--hpo_metric_save_path', - os.path.join(bucket, metric_path), '--model_dir', - os.path.join(bucket, - model_path), '--oss_host', environment['oss_endpoint'], - '--role_arn', environment['role_arn'], '--algo_proj_name', algo_proj_name + 'python', + '-m', + 'easy_rec.python.hpo.generate_hpo_sql', + '--sql_path', + sql_path, + '--config_path', + config_path, + '--cluster', + cluster, + '--bucket', + bucket, + '--hpo_param_path', + os.path.join(bucket, param_path), + '--hpo_metric_save_path', + os.path.join(bucket, metric_path), + '--model_dir', + os.path.join(bucket, model_path), + '--oss_host', + environment['oss_endpoint'], + '--role_arn', + environment['role_arn'], + '--algo_proj_name', + algo_proj_name, ] if tables: @@ -147,8 +176,8 @@ def _add_prefix(table_name): 'metric_reader': { 'type': 'oss_reader', 'location': metric_path, - 'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name - } + 'parser_pattern': '.*"%s": (\\d.\\d+).*' % metric_name, + }, } tasks = [adapter_task, prepare_sql_task, train_task] @@ -157,7 +186,7 @@ def _add_prefix(table_name): 'algorithm': algorithm, 'hyperparams': hyperparams, 'tasks': tasks, - 'environment': environment + 'environment': environment, } return data, tmp_dir @@ -188,7 +217,7 @@ def _add_prefix(table_name): '--cluster', type=str, help='cluster spec', - default='{"ps":{"count":1, "cpu":1000}, "worker" : {"count":3, "cpu":1000, "gpu":100, "memory":40000}}' + default='{"ps":{"count":1, "cpu":1000}, "worker" : {"count":3, "cpu":1000, "gpu":100, "memory":40000}}', ) parser.add_argument( '--algo_proj_name', @@ -205,7 +234,8 @@ def _add_prefix(table_name): '--max_parallel', type=int, help='max number of trials run at the same time', - default=4) + default=4, + ) parser.add_argument( '--total_trial_num', type=int, @@ -214,7 +244,8 @@ def _add_prefix(table_name): parser.add_argument( '--debug', action='store_true', - help='debug mode, will keep the temporary folder') + help='debug mode, will keep the temporary folder', + ) args = parser.parse_args() @@ -267,7 +298,7 @@ def _add_prefix(table_name): 'biz_id': '147331^paistudio^xxxxxxx^2020-03-18', 'role_arn': args.role_arn, 'bucket': args.bucket, - 'oss_endpoint': oss_config['endpoint'] + 'oss_endpoint': oss_config['endpoint'], } assert args.hyperparams is not None @@ -279,12 +310,21 @@ def _add_prefix(table_name): assert args.tables is not None or (args.train_tables is not None and args.eval_tables is not None) - data, tmp_dir = hpo_config(args.config_path, hyperparams, environment, - args.exp_dir, args.tables, args.train_tables, - args.eval_tables, args.cluster, - args.algo_proj_name, args.algo_res_proj, - args.algo_version, args.metric_name, - args.odps_config) + data, tmp_dir = hpo_config( + args.config_path, + hyperparams, + environment, + args.exp_dir, + args.tables, + args.train_tables, + args.eval_tables, + args.cluster, + args.algo_proj_name, + args.algo_res_proj, + args.algo_version, + args.metric_name, + args.odps_config, + ) hpo_util.kill_old_proc(tmp_dir, platform='pai') data_json = json.dumps(data) diff --git a/easy_rec/python/inference/client/client_demo.py b/easy_rec/python/inference/client/client_demo.py index 9464b1073..e6bd4df1f 100644 --- a/easy_rec/python/inference/client/client_demo.py +++ b/easy_rec/python/inference/client/client_demo.py @@ -74,7 +74,8 @@ def send_request(req_pb, client, debug_level=0): '--endpoint', type=str, default=None, - help='eas endpoint, such as 12345.cn-beijing.pai-eas.aliyuncs.com') + help='eas endpoint, such as 12345.cn-beijing.pai-eas.aliyuncs.com', + ) parser.add_argument( '--service_name', type=str, default=None, help='eas service name') parser.add_argument( diff --git a/easy_rec/python/inference/csv_predictor.py b/easy_rec/python/inference/csv_predictor.py index c5e154d2e..ae5fc1fcc 100644 --- a/easy_rec/python/inference/csv_predictor.py +++ b/easy_rec/python/inference/csv_predictor.py @@ -10,26 +10,30 @@ import tensorflow as tf from tensorflow.python.platform import gfile -from easy_rec.python.inference.predictor import SINGLE_PLACEHOLDER_FEATURE_KEY -from easy_rec.python.inference.predictor import Predictor from easy_rec.python.protos.dataset_pb2 import DatasetConfig from easy_rec.python.utils.check_utils import check_split +from easy_rec.python.inference.predictor import ( # NOQA + SINGLE_PLACEHOLDER_FEATURE_KEY, Predictor, +) + if tf.__version__ >= '2.0': tf = tf.compat.v1 class CSVPredictor(Predictor): - def __init__(self, - model_path, - data_config, - with_header=False, - ds_vector_recall=False, - fg_json_path=None, - profiling_file=None, - selected_cols=None, - output_sep=chr(1)): + def __init__( + self, + model_path, + data_config, + with_header=False, + ds_vector_recall=False, + fg_json_path=None, + profiling_file=None, + selected_cols=None, + output_sep=chr(1), + ): super(CSVPredictor, self).__init__(model_path, profiling_file, fg_json_path) self._output_sep = output_sep self._ds_vector_recall = ds_vector_recall @@ -74,16 +78,19 @@ def _get_reserved_cols(self, reserved_cols): def _parse_line(self, line): check_list = [ tf.py_func( - check_split, [line, self._input_sep, - len(self._record_defaults)], - Tout=tf.bool) + check_split, + [line, self._input_sep, + len(self._record_defaults)], + Tout=tf.bool, + ) ] with tf.control_dependencies(check_list): fields = tf.decode_csv( line, field_delim=self._input_sep, record_defaults=self._record_defaults, - name='decode_csv') + name='decode_csv', + ) if self._is_rtp: if self._with_header: inputs = dict(zip(self._all_fields, fields)) @@ -110,7 +117,7 @@ def _get_num_cols(self, file_paths): line_tok = line_str.strip().split(self._input_sep) if num_cols != -1: assert num_cols == len(line_tok), ( - 'num selected cols is %d, not equal to %d, current line is: %s, please check input_sep and data.' + 'num selected cols is %d, not equal to %d, current line is: %s, please check input_sep and data' % (num_cols, len(line_tok), line_str)) num_cols = len(line_tok) num_lines += 1 @@ -159,7 +166,8 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, dataset = dataset.interleave( lambda x: tf.data.TextLineDataset(x).skip(int(self._with_header)), cycle_length=parallel_num, - num_parallel_calls=parallel_num) + num_parallel_calls=parallel_num, + ) dataset = dataset.shard(slice_num, slice_id) dataset = dataset.batch(batch_size) dataset = dataset.prefetch(buffer_size=64) @@ -180,10 +188,10 @@ def _write_lines(self, table_writer, outputs): table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): - reserve_vals = [outputs[x] for x in output_cols] + \ - [all_vals[k] for k in reserved_cols] + reserve_vals = [outputs[x] for x in output_cols + ] + [all_vals[k] for k in reserved_cols] return reserve_vals @property def out_of_range_exception(self): - return (tf.errors.OutOfRangeError) + return tf.errors.OutOfRangeError diff --git a/easy_rec/python/inference/hive_parquet_predictor.py b/easy_rec/python/inference/hive_parquet_predictor.py index bd5178fe7..77cbf291b 100644 --- a/easy_rec/python/inference/hive_parquet_predictor.py +++ b/easy_rec/python/inference/hive_parquet_predictor.py @@ -24,15 +24,17 @@ class HiveParquetPredictor(Predictor): - def __init__(self, - model_path, - data_config, - hive_config, - fg_json_path=None, - profiling_file=None, - output_sep=chr(1), - all_cols=None, - all_col_types=None): + def __init__( + self, + model_path, + data_config, + hive_config, + fg_json_path=None, + profiling_file=None, + output_sep=chr(1), + all_cols=None, + all_col_types=None, + ): super(HiveParquetPredictor, self).__init__(model_path, profiling_file, fg_json_path) @@ -80,7 +82,7 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, 'double': tf.double, 'float': tf.float32, 'bigint': tf.int32, - 'boolean': tf.bool + 'boolean': tf.bool, } for col_name, col_type in zip(self._all_cols, self._all_col_types): if col_name in input_field_type_map: @@ -135,7 +137,9 @@ def _get_writer(self, output_path, slice_id): output_path = output_path.replace('.', '/') self._hdfs_path = 'hdfs://%s:9000/user/easy_rec/%s_tmp' % ( - self._hive_config.host, output_path) + self._hive_config.host, + output_path, + ) if not gfile.Exists(self._hdfs_path): gfile.MakeDirs(self._hdfs_path) res_path = os.path.join(self._hdfs_path, 'part-%d.csv' % slice_id) @@ -148,8 +152,8 @@ def _write_lines(self, table_writer, outputs): table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): - reserve_vals = [outputs[x] for x in output_cols] + \ - [all_vals[k] for k in reserved_cols] + reserve_vals = [outputs[x] for x in output_cols + ] + [all_vals[k] for k in reserved_cols] return reserve_vals def load_to_table(self, output_path, slice_num, slice_id): @@ -181,20 +185,28 @@ def load_to_table(self, output_path, slice_num, slice_id): schema = schema.rstrip(',') if partition_name and partition_val: - sql = 'create table if not exists %s (%s) PARTITIONED BY (%s string)' % \ - (table_name, schema, partition_name) + sql = 'create table if not exists %s (%s) PARTITIONED BY (%s string)' % ( + table_name, + schema, + partition_name, + ) self._hive_util.run_sql(sql) - sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s PARTITION (%s=%s)" % \ - (self._hdfs_path, table_name, partition_name, partition_val) + sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s PARTITION (%s=%s)" % ( + self._hdfs_path, + table_name, + partition_name, + partition_val, + ) self._hive_util.run_sql(sql) else: - sql = 'create table if not exists %s (%s)' % \ - (table_name, schema) + sql = 'create table if not exists %s (%s)' % (table_name, schema) self._hive_util.run_sql(sql) - sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s" % \ - (self._hdfs_path, table_name) + sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s" % ( + self._hdfs_path, + table_name, + ) self._hive_util.run_sql(sql) @property def out_of_range_exception(self): - return (tf.errors.OutOfRangeError) + return tf.errors.OutOfRangeError diff --git a/easy_rec/python/inference/hive_predictor.py b/easy_rec/python/inference/hive_predictor.py index f2923f8a3..043bf5463 100644 --- a/easy_rec/python/inference/hive_predictor.py +++ b/easy_rec/python/inference/hive_predictor.py @@ -21,15 +21,17 @@ class HivePredictor(Predictor): - def __init__(self, - model_path, - data_config, - hive_config, - fg_json_path=None, - profiling_file=None, - output_sep=chr(1), - all_cols=None, - all_col_types=None): + def __init__( + self, + model_path, + data_config, + hive_config, + fg_json_path=None, + profiling_file=None, + output_sep=chr(1), + all_cols=None, + all_col_types=None, + ): super(HivePredictor, self).__init__(model_path, profiling_file, fg_json_path) @@ -61,7 +63,8 @@ def _parse_line(self, line): line, field_delim=field_delim, record_defaults=self._record_defaults, - name='decode_csv') + name='decode_csv', + ) inputs = {self._all_cols[x]: fields[x] for x in range(len(fields))} return inputs @@ -78,7 +81,8 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, dataset = dataset.interleave( tf.data.TextLineDataset, cycle_length=parallel_num, - num_parallel_calls=parallel_num) + num_parallel_calls=parallel_num, + ) dataset = dataset.shard(slice_num, slice_id) dataset = dataset.batch(batch_size) dataset = dataset.prefetch(buffer_size=64) @@ -101,7 +105,9 @@ def _get_writer(self, output_path, slice_id): output_path = output_path.replace('.', '/') self._hdfs_path = 'hdfs://%s:9000/user/easy_rec/%s_tmp' % ( - self._hive_config.host, output_path) + self._hive_config.host, + output_path, + ) if not gfile.Exists(self._hdfs_path): gfile.MakeDirs(self._hdfs_path) res_path = os.path.join(self._hdfs_path, 'part-%d.csv' % slice_id) @@ -114,8 +120,8 @@ def _write_lines(self, table_writer, outputs): table_writer.write(outputs + '\n') def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): - reserve_vals = [outputs[x] for x in output_cols] + \ - [all_vals[k] for k in reserved_cols] + reserve_vals = [outputs[x] for x in output_cols + ] + [all_vals[k] for k in reserved_cols] return reserve_vals def load_to_table(self, output_path, slice_num, slice_id): @@ -147,20 +153,28 @@ def load_to_table(self, output_path, slice_num, slice_id): schema = schema.rstrip(',') if partition_name and partition_val: - sql = 'create table if not exists %s (%s) PARTITIONED BY (%s string)' % \ - (table_name, schema, partition_name) + sql = 'create table if not exists %s (%s) PARTITIONED BY (%s string)' % ( + table_name, + schema, + partition_name, + ) self._hive_util.run_sql(sql) - sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s PARTITION (%s=%s)" % \ - (self._hdfs_path, table_name, partition_name, partition_val) + sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s PARTITION (%s=%s)" % ( + self._hdfs_path, + table_name, + partition_name, + partition_val, + ) self._hive_util.run_sql(sql) else: - sql = 'create table if not exists %s (%s)' % \ - (table_name, schema) + sql = 'create table if not exists %s (%s)' % (table_name, schema) self._hive_util.run_sql(sql) - sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s" % \ - (self._hdfs_path, table_name) + sql = "LOAD DATA INPATH '%s/*' INTO TABLE %s" % ( + self._hdfs_path, + table_name, + ) self._hive_util.run_sql(sql) @property def out_of_range_exception(self): - return (tf.errors.OutOfRangeError) + return tf.errors.OutOfRangeError diff --git a/easy_rec/python/inference/odps_predictor.py b/easy_rec/python/inference/odps_predictor.py index 183fc4d13..1cdb4b9e2 100644 --- a/easy_rec/python/inference/odps_predictor.py +++ b/easy_rec/python/inference/odps_predictor.py @@ -11,12 +11,14 @@ class ODPSPredictor(Predictor): - def __init__(self, - model_path, - fg_json_path=None, - profiling_file=None, - all_cols='', - all_col_types=''): + def __init__( + self, + model_path, + fg_json_path=None, + profiling_file=None, + all_cols='', + all_col_types='', + ): super(ODPSPredictor, self).__init__(model_path, profiling_file, fg_json_path) self._all_cols = [x.strip() for x in all_cols.split(',') if x != ''] @@ -45,13 +47,15 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, record_defaults=self._record_defaults, slice_id=slice_id, slice_count=slice_num, - selected_cols=','.join(self._all_cols)) + selected_cols=','.join(self._all_cols), + ) dataset = dataset.batch(batch_size) dataset = dataset.prefetch(buffer_size=64) return dataset def _get_writer(self, output_path, slice_id): import common_io + table_writer = common_io.table.TableWriter(output_path, slice_id=slice_id) return table_writer @@ -65,6 +69,6 @@ def out_of_range_exception(self): return (tf.python_io.OutOfRangeException, tf.errors.OutOfRangeError) def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): - reserve_vals = [all_vals[k] for k in reserved_cols] + \ - [outputs[x] for x in output_cols] + reserve_vals = [all_vals[k] for k in reserved_cols + ] + [outputs[x] for x in output_cols] return reserve_vals diff --git a/easy_rec/python/inference/parquet_predictor.py b/easy_rec/python/inference/parquet_predictor.py index 7fead6388..7f59e6573 100644 --- a/easy_rec/python/inference/parquet_predictor.py +++ b/easy_rec/python/inference/parquet_predictor.py @@ -20,7 +20,9 @@ try: from tensorflow.python.framework.load_library import load_op_library + import easy_rec + load_embed_lib_path = os.path.join(easy_rec.ops_dir, 'libload_embed.so') load_embed_lib = load_op_library(load_embed_lib_path) except Exception as ex: @@ -29,15 +31,17 @@ class ParquetPredictor(Predictor): - def __init__(self, - model_path, - data_config, - ds_vector_recall=False, - fg_json_path=None, - profiling_file=None, - selected_cols=None, - output_sep=chr(1), - pipeline_config=None): + def __init__( + self, + model_path, + data_config, + ds_vector_recall=False, + fg_json_path=None, + profiling_file=None, + selected_cols=None, + output_sep=chr(1), + pipeline_config=None, + ): super(ParquetPredictor, self).__init__(model_path, profiling_file, fg_json_path) self._output_sep = output_sep @@ -98,9 +102,10 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, parquet_file = gfile.Glob(input_path.split(',')[0])[0] kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file( self._reserved_cols, parquet_file) - logging.info('reserve_fields=%s reserve_types=%s' % - (','.join(self._reserved_cols), ','.join( - [str(x) for x in kwargs['reserve_types']]))) + logging.info('reserve_fields=%s reserve_types=%s' % ( + ','.join(self._reserved_cols), + ','.join([str(x) for x in kwargs['reserve_types']]), + )) else: self._reserved_cols = [] self.pipeline_config.data_config.batch_size = batch_size @@ -113,7 +118,8 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, task_index=slice_id, task_num=slice_num, pipeline_config=self.pipeline_config, - **kwargs) + **kwargs, + ) return parquet_input._build(tf.estimator.ModeKeys.PREDICT, {}) def _get_writer(self, output_path, slice_id): @@ -144,4 +150,4 @@ def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): @property def out_of_range_exception(self): - return (tf.errors.OutOfRangeError) + return tf.errors.OutOfRangeError diff --git a/easy_rec/python/inference/parquet_predictor_v2.py b/easy_rec/python/inference/parquet_predictor_v2.py index 1ee08517b..c3dbf3078 100644 --- a/easy_rec/python/inference/parquet_predictor_v2.py +++ b/easy_rec/python/inference/parquet_predictor_v2.py @@ -20,7 +20,9 @@ try: from tensorflow.python.framework.load_library import load_op_library + import easy_rec + load_embed_lib_path = os.path.join(easy_rec.ops_dir, 'libload_embed.so') load_embed_lib = load_op_library(load_embed_lib_path) except Exception as ex: @@ -29,15 +31,17 @@ class ParquetPredictorV2(Predictor): - def __init__(self, - model_path, - data_config, - ds_vector_recall=False, - fg_json_path=None, - profiling_file=None, - selected_cols=None, - output_sep=chr(1), - pipeline_config=None): + def __init__( + self, + model_path, + data_config, + ds_vector_recall=False, + fg_json_path=None, + profiling_file=None, + selected_cols=None, + output_sep=chr(1), + pipeline_config=None, + ): super(ParquetPredictorV2, self).__init__(model_path, profiling_file, fg_json_path) self._output_sep = output_sep @@ -98,9 +102,10 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, parquet_file = gfile.Glob(input_path.split(',')[0])[0] kwargs['reserve_types'] = input_utils.get_tf_type_from_parquet_file( self._reserved_cols, parquet_file) - logging.info('reserve_fields=%s reserve_types=%s' % - (','.join(self._reserved_cols), ','.join( - [str(x) for x in kwargs['reserve_types']]))) + logging.info('reserve_fields=%s reserve_types=%s' % ( + ','.join(self._reserved_cols), + ','.join([str(x) for x in kwargs['reserve_types']]), + )) else: self._reserved_cols = [] self.pipeline_config.data_config.batch_size = batch_size @@ -113,7 +118,8 @@ def _get_dataset(self, input_path, num_parallel_calls, batch_size, slice_num, task_index=slice_id, task_num=slice_num, pipeline_config=self.pipeline_config, - **kwargs) + **kwargs, + ) return parquet_input._build(tf.estimator.ModeKeys.PREDICT, {}) def _get_writer(self, output_path, slice_id): @@ -144,4 +150,4 @@ def _get_reserve_vals(self, reserved_cols, output_cols, all_vals, outputs): @property def out_of_range_exception(self): - return (tf.errors.OutOfRangeError) + return tf.errors.OutOfRangeError diff --git a/easy_rec/python/inference/predictor.py b/easy_rec/python/inference/predictor.py index 2d02fac71..618789266 100644 --- a/easy_rec/python/inference/predictor.py +++ b/easy_rec/python/inference/predictor.py @@ -21,12 +21,13 @@ import easy_rec from easy_rec.python.utils import numpy_utils -from easy_rec.python.utils.config_util import get_configs_from_pipeline_file -from easy_rec.python.utils.config_util import get_input_name_from_fg_json -from easy_rec.python.utils.config_util import search_fg_json from easy_rec.python.utils.input_utils import get_type_defaults from easy_rec.python.utils.load_class import get_register_class_meta +from easy_rec.python.utils.config_util import ( # NOQA + get_configs_from_pipeline_file, get_input_name_from_fg_json, search_fg_json, +) + try: tf.load_op_library(os.path.join(easy_rec.ops_dir, 'libcustom_ops.so')) except Exception as ex: @@ -52,7 +53,6 @@ def __init__(self, model_path, model_config=None): model_path: init model from this directory model_config: config string for model to init, in json format """ - pass @abc.abstractmethod def predict(self, input_data, batch_size): @@ -68,7 +68,6 @@ def predict(self, input_data, batch_size): eg, {"output1": value1, "output2": value2}, the value type can be python int str float, and numpy array """ - pass def get_output_type(self): """Get output types of prediction. @@ -83,7 +82,8 @@ def get_output_type(self): * type image, data will be converted to encode image binary and write to oss file, whose name is output_dir/${key}/${input_filename}_${idx}.jpg, where input_filename is extracted from url, key corresponds to the key in the dict of output_type, - if the type of data indexed by key is a list, idx is the index of element in list, otherwhile ${idx} will be empty + if the type of data indexed by key is a list, idx is the index of element in list, + otherwhile ${idx} will be empty * type video, data will be converted to encode video binary and write to oss file, @@ -157,7 +157,8 @@ def search_pb(self, directory): logging.info('find %d models: %s' % (len(dir_list), ','.join(dir_list))) dir_list = sorted( dir_list, - key=lambda x: int(x.split('/')[(-2 if (x[-1] == '/') else -1)])) + key=lambda x: int(x.split('/')[(-2 if (x[-1] == '/') else -1)]), + ) return dir_list[-1] else: raise ValueError('multiple saved model found in directory %s' % @@ -191,7 +192,8 @@ def _build_model(self): session_config = tf.ConfigProto( gpu_options=gpu_options, allow_soft_placement=True, - log_device_placement=(self._profiling_file is not None)) + log_device_placement=(self._profiling_file is not None), + ) self._session = tf.Session(config=session_config, graph=self._graph) with self._graph.as_default(): @@ -202,13 +204,18 @@ def _build_model(self): if gfile.IsDirectory(model_path): model_path = self.search_pb(model_path) logging.info('model find in %s' % model_path) - self._input_fields_info, self._input_fields_list = self._get_input_fields_from_pipeline_config( - model_path) - assert tf.saved_model.loader.maybe_saved_model_directory(model_path), \ - 'saved model does not exists in %s' % model_path + ( + self._input_fields_info, + self._input_fields_list, + ) = self._get_input_fields_from_pipeline_config(model_path) + assert tf.saved_model.loader.maybe_saved_model_directory( + model_path), ('saved model does not exists in %s' % model_path) self._is_saved_model = True meta_graph_def = tf.saved_model.loader.load( - self._session, [tf.saved_model.tag_constants.SERVING], model_path) + self._session, + [tf.saved_model.tag_constants.SERVING], + model_path, + ) # parse signature signature_def = meta_graph_def.signature_def[ signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] @@ -263,9 +270,10 @@ def _build_model(self): type_name = asset_file.tensor_info.name.split(':')[0] asset_path = os.path.join(model_path, constants.ASSETS_DIRECTORY, asset_file.filename) - assert gfile.Exists( - asset_path), '%s is missing in saved model' % asset_path - self._assets[type_name] = asset_path + # assert gfile.Exists( + # asset_path), '%s is missing in saved model' % asset_path + if gfile.Exists(asset_path): + self._assets[type_name] = asset_path logging.info(self._assets) # get export config @@ -275,7 +283,7 @@ def _build_model(self): # self._export_config = json.loads(export_config_collection[0]) # logging.info('load export config info %s' % export_config_collection[0]) else: - raise ValueError('currently only savedmodel is supported') + raise ValueError('currently only saved_model is supported') def predict(self, input_data_dict, output_names=None): """Predict input data with loaded model. @@ -294,15 +302,18 @@ def predict(self, input_data_dict, output_names=None): assert input_name in input_data_dict, 'input data %s is missing' % input_name tensor_shape = tensor.get_shape().as_list() input_shape = input_data_dict[input_name].shape - assert tensor_shape[0] is None or (tensor_shape[0] == input_shape[0]), \ - 'input %s batchsize %d is not the same as the exported batch_size %d' % \ - (input_name, input_shape[0], tensor_shape[0]) + assert tensor_shape[0] is None or (tensor_shape[0] == input_shape[0]), ( + 'input %s batchsize %d is not the same as the exported batch_size %d' + % ( + input_name, + input_shape[0], + tensor_shape[0], + )) feed_dict[tensor] = input_data_dict[input_name] fetch_dict = {} if output_names is not None: for output_name in output_names: - assert output_name in self._outputs_map, \ - 'invalid output name %s' % output_name + assert output_name in self._outputs_map, 'invalid output name %s' % output_name fetch_dict[output_name] = self._outputs_map[output_name] else: fetch_dict = self._outputs_map @@ -318,9 +329,11 @@ def predict(self, input_data_dict, output_names=None): fetch_dict, feed_dict, options=run_options, - run_metadata=run_metadata) + run_metadata=run_metadata, + ) # Create the Timeline object, and write it to a json from tensorflow.python.client import timeline + tl = timeline.Timeline(run_metadata.step_stats) ctf = tl.generate_chrome_trace_format() with gfile.GFile(self._profiling_file, 'w') as f: @@ -384,7 +397,9 @@ def _get_defaults(self, col_name, col_type='string'): else: defaults = {'string': '', 'double': 0.0, 'bigint': 0} assert col_type in defaults, 'invalid col_type: %s, col_type: %s' % ( - col_name, col_type) + col_name, + col_type, + ) default_val = defaults[col_type] logging.info( 'col_name: %s, default_val: %s.[not defined in saved_model_dir/assets/pipeline.config]' @@ -548,9 +563,9 @@ def _parse_value(all_vals): ts3 = time.time() progress += 1 - sum_t0 += (ts1 - ts0) - sum_t1 += (ts2 - ts1) - sum_t2 += (ts3 - ts2) + sum_t0 += ts1 - ts0 + sum_t1 += ts2 - ts1 + sum_t2 += ts3 - ts2 except self.out_of_range_exception: break if progress % 100 == 0: @@ -577,9 +592,10 @@ def predict(self, input_data_dict_list, output_names=None, batch_size=1): """ num_example = len(input_data_dict_list) assert num_example > 0, 'input data should not be an empty list' - assert isinstance(input_data_dict_list[0], dict) or \ - isinstance(input_data_dict_list[0], list) or \ - isinstance(input_data_dict_list[0], str), 'input is not a list or dict or str' + assert (isinstance(input_data_dict_list[0], dict) or + isinstance(input_data_dict_list[0], list) or + isinstance(input_data_dict_list[0], + str)), 'input is not a list or dict or str' if batch_size > 0: num_batches = int(math.ceil(float(num_example) / batch_size)) else: @@ -608,9 +624,11 @@ def batch(self, data_list): for key in data: batch_input[key].append(data[key]) elif isinstance(data, list): - assert len(self._predictor_impl.input_names) == len(data), \ - 'input fields number incorrect, should be %d, but %d' \ - % (len(self._predictor_impl.input_names), len(data)) + assert len(self._predictor_impl.input_names) == len(data), ( + 'input fields number incorrect, should be %d, but %d' % ( + len(self._predictor_impl.input_names), + len(data), + )) for key, v in zip(self._predictor_impl.input_names, data): if key != '': batch_input[key].append(v) diff --git a/easy_rec/python/inference/processor/test.py b/easy_rec/python/inference/processor/test.py index 088c93edc..df24edbdd 100644 --- a/easy_rec/python/inference/processor/test.py +++ b/easy_rec/python/inference/processor/test.py @@ -79,8 +79,8 @@ def build_array_proto(array_proto, data, dtype): assert os.path.exists(args.libc_path), '%s does not exist' % args.libc_path assert args.saved_model_dir is not None and os.path.isdir( - args.saved_model_dir - ), '%s is not a valid directory' % args.saved_model_dir + args.saved_model_dir), ('%s is not a valid directory' % + args.saved_model_dir) assert args.input_path is not None and os.path.exists( args.input_path), '%s does not exist' % args.input_path assert args.output_path is not None, 'output_path is not set' @@ -108,9 +108,11 @@ def build_array_proto(array_proto, data, dtype): req = tf_predict_pb2.PredictRequest() req.signature_name = 'serving_default' for i in range(len(input_fields)): - build_array_proto(req.inputs[data_config.input_fields[i + 1].input_name], - input_fields[i], - data_config.input_fields[i + 1].input_type) + build_array_proto( + req.inputs[data_config.input_fields[i + 1].input_name], + input_fields[i], + data_config.input_fields[i + 1].input_type, + ) tf_predictor = ctypes.cdll.LoadLibrary(PROCESSOR_ENTRY_LIB) tf_predictor.saved_model_init.restype = ctypes.c_void_p @@ -152,8 +154,11 @@ def build_array_proto(array_proto, data, dtype): tf_predictor.saved_model_predict.restype = ctypes.c_void_p out_len = ctypes.c_int(0) res_p = tf_predictor.saved_model_predict( - ctypes.c_void_p(handle), data_bin, ctypes.c_int32(len(data_bin)), - ctypes.byref(out_len)) + ctypes.c_void_p(handle), + data_bin, + ctypes.c_int32(len(data_bin)), + ctypes.byref(out_len), + ) res_bytes = bytearray(ctypes.string_at(res_p, out_len)) res = tf_predict_pb2.PredictResponse() res.ParseFromString(res_bytes) diff --git a/easy_rec/python/inference/vector_retrieve.py b/easy_rec/python/inference/vector_retrieve.py index a8a8122d5..37cf220bf 100644 --- a/easy_rec/python/inference/vector_retrieve.py +++ b/easy_rec/python/inference/vector_retrieve.py @@ -24,18 +24,20 @@ class VectorRetrieve(object): - def __init__(self, - query_table, - doc_table, - out_table, - ndim, - delimiter=',', - batch_size=4, - index_type='ivfflat', - nlist=10, - nprobe=2, - distance=1, - m=8): + def __init__( + self, + query_table, + doc_table, + out_table, + ndim, + delimiter=',', + batch_size=4, + index_type='ivfflat', + nlist=10, + nprobe=2, + distance=1, + m=8, + ): """Retrieve top n neighbours by query vector. Args: @@ -75,7 +77,8 @@ def __call__(self, top_n, task_index, task_count, *args, **kwargs): 'doc', decoder=gl.Decoder( attr_types=['float'] * self.ndim, attr_delimiter=self.delimiter), - option=self.knn_option) + option=self.knn_option, + ) g.init(task_index=task_index, task_count=task_count) query_reader = common_io.table.TableReader( @@ -121,4 +124,5 @@ def __call__(self, top_n, task_index, task_count, *args, **kwargs): def to_np_array(batch_query_feats, attr_delimiter): return np.array( [map(float, feat.split(attr_delimiter)) for feat in batch_query_feats], - dtype='float32') + dtype='float32', + ) diff --git a/easy_rec/python/input/batch_tfrecord_input.py b/easy_rec/python/input/batch_tfrecord_input.py index fb1981f60..77af079a8 100644 --- a/easy_rec/python/input/batch_tfrecord_input.py +++ b/easy_rec/python/input/batch_tfrecord_input.py @@ -19,23 +19,35 @@ class BatchTFRecordInput(Input): batch_size needs to be a multiple of n. """ - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(BatchTFRecordInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(BatchTFRecordInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) assert data_config.HasField( 'n_data_batch_tfrecord'), 'Need to set n_data_batch_tfrecord in config.' self._input_shapes = [x.input_shape for x in data_config.input_fields] self.feature_desc = {} - for x, t, d, s in zip(self._input_fields, self._input_field_types, - self._input_field_defaults, self._input_shapes): + for x, t, d, s in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + self._input_shapes, + ): d = self.get_type_defaults(t, d) t = get_tf_type(t) self.feature_desc[x] = tf.io.FixedLenSequenceFeature( @@ -49,12 +61,15 @@ def _parse_tfrecord(self, example): _, features, _ = tf.io.parse_sequence_example( example, sequence_features=self.feature_desc) # Below code will reduce one dimension when the data dimension > 2. - features = dict( - (key, - tf.reshape(value, [ - -1, - ] + [x for i, x in enumerate(value.shape) if i not in (0, 1)])) for ( - key, value) in features.items()) + features = dict(( + key, + tf.reshape( + value, + [ + -1, + ] + [x for i, x in enumerate(value.shape) if i not in (0, 1)], + ), + ) for (key, value) in features.items()) return features def _build(self, mode, params): @@ -81,13 +96,15 @@ def _build(self, mode, params): lambda x: tf.data.TFRecordDataset( x, compression_type=data_compression_type), cycle_length=parallel_num, - num_parallel_calls=parallel_num) + num_parallel_calls=parallel_num, + ) dataset = dataset.shard(self._task_num, self._task_index) if self._data_config.shuffle: dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True) + reshuffle_each_iteration=True, + ) dataset = dataset.repeat(self.num_epochs) else: logging.info('eval files[%d]: %s' % diff --git a/easy_rec/python/input/criteo_binary_reader.py b/easy_rec/python/input/criteo_binary_reader.py index 6672165c0..c27b4aae3 100644 --- a/easy_rec/python/input/criteo_binary_reader.py +++ b/easy_rec/python/input/criteo_binary_reader.py @@ -61,8 +61,8 @@ def __init__( def _compute_global_start_pos(self, total_sample_num, batch_size, global_rank, global_size, drop_last): # ensure all workers have the same number of samples - avg_sample_num = (total_sample_num // global_size) - res_num = (total_sample_num % global_size) + avg_sample_num = total_sample_num // global_size + res_num = total_sample_num % global_size self._num_samples = avg_sample_num if res_num > 0: self._num_samples += 1 @@ -174,15 +174,21 @@ def _get(self, idx): label_raw_data, dtype=np.int32).reshape([tmp_read_num, 1]) label_read_arr.append(tmp_lbl_np) - dense_raw_data = os.pread(self._dense_file_arr[curr_file_id], - 52 * tmp_read_num, start_read_pos * 52) + dense_raw_data = os.pread( + self._dense_file_arr[curr_file_id], + 52 * tmp_read_num, + start_read_pos * 52, + ) part_dense_np = np.frombuffer( dense_raw_data, dtype=np.float32).reshape([tmp_read_num, 13]) # part_dense_np = np.log(part_dense_np + 3, dtype=np.float32) dense_read_arr.append(part_dense_np) - category_raw_data = os.pread(self._category_file_arr[curr_file_id], - 104 * tmp_read_num, start_read_pos * 104) + category_raw_data = os.pread( + self._category_file_arr[curr_file_id], + 104 * tmp_read_num, + start_read_pos * 104, + ) part_cate_np = np.frombuffer( category_raw_data, dtype=np.uint32).reshape([tmp_read_num, 26]) cate_read_arr.append(part_cate_np) diff --git a/easy_rec/python/input/criteo_input.py b/easy_rec/python/input/criteo_input.py index 0eb3ee595..00412a4c9 100644 --- a/easy_rec/python/input/criteo_input.py +++ b/easy_rec/python/input/criteo_input.py @@ -13,26 +13,39 @@ class CriteoInput(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(CriteoInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(CriteoInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) all_label_paths = [] all_dense_paths = [] all_category_paths = [] if input_path is not None: - assert len(input_path.label_path) == len(input_path.dense_path) and \ - len(input_path.label_path) == len(input_path.category_path), \ - 'label_path_num(%d), dense_path_num(%d), category_path_num(%d) must be the same' % \ - (len(input_path.label_path), len(input_path.dense_path), len(input_path.category_path)) + assert len(input_path.label_path) == len(input_path.dense_path) and len( + input_path.label_path + ) == len( + input_path.category_path + ), 'label_path_num(%d), dense_path_num(%d), category_path_num(%d) must be the same' % ( + len(input_path.label_path), + len(input_path.dense_path), + len(input_path.category_path), + ) for label_path, dense_path, category_path in zip( input_path.label_path, input_path.dense_path, @@ -40,10 +53,11 @@ def __init__(self, label_paths = tf.gfile.Glob(input_path.label_path) dense_paths = tf.gfile.Glob(input_path.dense_path) category_paths = tf.gfile.Glob(input_path.category_path) - assert len(label_paths) == len(dense_paths) and len(label_paths) == \ - len(category_paths), 'label_path(%s) dense_path(%s) category_path(%s) ' + \ - 'matched different number of files(%d %d %d)' % ( - len(label_paths), len(dense_paths), len(category_paths)) + assert len(label_paths) == len(dense_paths) and len(label_paths) == len( + category_paths), ( + 'label_path(%s) dense_path(%s) category_path(%s) ' + + 'matched different number of files(%d %d %d)' % + (len(label_paths), len(dense_paths), len(category_paths))) label_paths.sort() dense_paths.sort() category_paths.sort() @@ -60,7 +74,8 @@ def __init__(self, self._batch_size, prefetch=self._prefetch_size, global_rank=self._task_index, - global_size=self._task_num) + global_size=self._task_num, + ) else: self._binary_reader = None @@ -89,8 +104,12 @@ def _build(self, mode, params): dataset = tf.data.Dataset.from_generator( self._sample_generator, output_types=(tf.float32, tf.int32, tf.int32), - output_shapes=(tf.TensorShape([None, 13]), tf.TensorShape([None, 26]), - tf.TensorShape([None]))) + output_shapes=( + tf.TensorShape([None, 13]), + tf.TensorShape([None, 26]), + tf.TensorShape([None]), + ), + ) num_parallel_calls = self._data_config.num_parallel_calls dataset = dataset.map( self._to_fea_dict, num_parallel_calls=num_parallel_calls) diff --git a/easy_rec/python/input/csv_input.py b/easy_rec/python/input/csv_input.py index b3afd1656..d4e974f49 100644 --- a/easy_rec/python/input/csv_input.py +++ b/easy_rec/python/input/csv_input.py @@ -16,17 +16,25 @@ class CSVInput(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(CSVInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(CSVInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) self._with_header = data_config.with_header self._field_names = None @@ -43,25 +51,32 @@ def _parse_csv(self, line): if field_name in self._input_fields: tid = self._input_fields.index(field_name) record_defaults.append( - self.get_type_defaults(self._input_field_types[tid], - self._input_field_defaults[tid])) + self.get_type_defaults( + self._input_field_types[tid], + self._input_field_defaults[tid], + )) else: record_defaults.append('') - check_list = [ + check_list = ([ tf.py_func( - check_split, [ - line, self._data_config.separator, - len(record_defaults), self._check_mode + check_split, + [ + line, + self._data_config.separator, + len(record_defaults), + self._check_mode, ], - Tout=tf.bool) - ] if self._check_mode else [] + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): fields = tf.decode_csv( line, field_delim=self._data_config.separator, record_defaults=record_defaults, - name='decode_csv') + name='decode_csv', + ) if self._field_names is not None: fields = [ fields[self._field_names.index(x)] for x in self._input_fields @@ -121,7 +136,8 @@ def _build(self, mode, params): x, compression_type=compression_type).skip( int(self._with_header)), cycle_length=parallel_num, - num_parallel_calls=parallel_num) + num_parallel_calls=parallel_num, + ) if not self._data_config.file_shard: dataset = self._safe_shard(dataset) @@ -130,7 +146,8 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True) + reshuffle_each_iteration=True, + ) dataset = dataset.repeat(self.num_epochs) elif self._task_num > 1: # For distribute evaluate dataset = tf.data.Dataset.from_tensor_slices(file_paths) @@ -140,7 +157,8 @@ def _build(self, mode, params): x, compression_type=compression_type).skip( int(self._with_header)), cycle_length=parallel_num, - num_parallel_calls=parallel_num) + num_parallel_calls=parallel_num, + ) dataset = self._safe_shard(dataset) dataset = dataset.repeat(1) else: @@ -153,7 +171,8 @@ def _build(self, mode, params): x, compression_type=compression_type).skip( int(self._with_header)), cycle_length=parallel_num, - num_parallel_calls=parallel_num) + num_parallel_calls=parallel_num, + ) dataset = dataset.repeat(1) dataset = dataset.batch(self._data_config.batch_size) diff --git a/easy_rec/python/input/csv_input_ex.py b/easy_rec/python/input/csv_input_ex.py index d3b506fce..4235ac09e 100644 --- a/easy_rec/python/input/csv_input_ex.py +++ b/easy_rec/python/input/csv_input_ex.py @@ -12,17 +12,25 @@ class CSVInputEx(CSVInput): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(CSVInputEx, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(CSVInputEx, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) def _parse_csv(self, line): record_defaults = [ @@ -35,9 +43,13 @@ def _check_data(line): if type(sep) != type(str): sep = sep.encode('utf-8') field_num = len(line[0].split(sep)) - assert field_num == len(record_defaults), \ - 'sep[%s] maybe invalid: field_num=%d, required_num=%d' % \ - (sep, field_num, len(record_defaults)) + assert field_num == len( + record_defaults + ), 'sep[%s] maybe invalid: field_num=%d, required_num=%d' % ( + sep, + field_num, + len(record_defaults), + ) return True fields = str_split_by_chr( @@ -53,13 +65,14 @@ def _check_data(line): fields.append( tf.string_to_number( tmp_fields[:, i], tf.float32, name='field_as_flt_%d' % i)) - elif type(record_defaults[i]) in [str, type(u''), bytes]: + elif type(record_defaults[i]) in [str, type(''), bytes]: fields.append(tmp_fields[:, i]) elif type(record_defaults[i]) == bool: fields.append( tf.logical_or( tf.equal(tmp_fields[:, i], 'True'), - tf.equal(tmp_fields[:, i], 'true'))) + tf.equal(tmp_fields[:, i], 'true'), + )) else: assert 'invalid types: %s' % str(type(record_defaults[i])) diff --git a/easy_rec/python/input/csv_input_v2.py b/easy_rec/python/input/csv_input_v2.py index deddc2f06..4495f95de 100644 --- a/easy_rec/python/input/csv_input_v2.py +++ b/easy_rec/python/input/csv_input_v2.py @@ -7,17 +7,25 @@ class CSVInputV2(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(CSVInputV2, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(CSVInputV2, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) def _build(self, mode, params): if type(self._input_path) != list: @@ -30,7 +38,7 @@ def _build(self, mode, params): dataset = tf.data.TextLineDataset(self._input_path) else: num_epochs = self.num_epochs if mode == tf.estimator.ModeKeys.TRAIN else 1 - is_train = (mode == tf.estimator.ModeKeys.TRAIN) + is_train = mode == tf.estimator.ModeKeys.TRAIN record_defaults = [ self.get_type_defaults(x, v) for x, v in zip(self._input_field_types, self._input_field_defaults) @@ -45,7 +53,8 @@ def _build(self, mode, params): num_epochs=num_epochs, shuffle=is_train and self._data_config.shuffle, num_parallel_reads=8, - sloppy=is_train) + sloppy=is_train, + ) if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.chief_redundant: diff --git a/easy_rec/python/input/datahub_input.py b/easy_rec/python/input/datahub_input.py index 37e3292d4..2a19bd92a 100644 --- a/easy_rec/python/input/datahub_input.py +++ b/easy_rec/python/input/datahub_input.py @@ -22,11 +22,11 @@ common_io = None try: + import urllib3 from datahub import DataHub from datahub.exceptions import DatahubException - from datahub.models import RecordType - from datahub.models import CursorType - import urllib3 + from datahub.models import CursorType, RecordType + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) logging.getLogger('datahub.account').setLevel(logging.INFO) except Exception: @@ -39,20 +39,30 @@ class DataHubInput(Input): """DataHubInput is used for online train.""" - def __init__(self, - data_config, - feature_config, - datahub_config, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(DataHubInput, - self).__init__(data_config, feature_config, '', task_index, task_num, - check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + datahub_config, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(DataHubInput, self).__init__( + data_config, + feature_config, + '', + task_index, + task_num, + check_mode, + pipeline_config, + ) if DataHub is None: - logging.error('please install datahub: ', - 'pip install pydatahub ;Python 3.6 recommended') + logging.error( + 'please install datahub: ', + 'pip install pydatahub ;Python 3.6 recommended', + ) try: self._num_epoch = 0 self._datahub_config = datahub_config @@ -69,7 +79,6 @@ def __init__(self, self._datahub = None except Exception as ex: logging.info('exception in init datahub: %s' % str(ex)) - pass self._offset_dict = {} if datahub_config: shard_result = self._datahub.list_shard(self._datahub_config.project, @@ -86,10 +95,13 @@ def __init__(self, ts = parse_time(datahub_config.offset_time) * 1000 for x in self._all_shards: ks = str(x.shard_id) - cursor_result = self._datahub.get_cursor(self._datahub_config.project, - self._datahub_config.topic, - ks, CursorType.SYSTEM_TIME, - ts) + cursor_result = self._datahub.get_cursor( + self._datahub_config.project, + self._datahub_config.topic, + ks, + CursorType.SYSTEM_TIME, + ts, + ) logging.info('shard[%s] cursor = %s' % (ks, cursor_result)) self._offset_dict[ks] = cursor_result.cursor elif offset_type == 'offset_info': @@ -101,7 +113,8 @@ def __init__(self, self._dh_field_types = [] topic_info = self._datahub.get_topic( project_name=self._datahub_config.project, - topic_name=self._datahub_config.topic) + topic_name=self._datahub_config.topic, + ) for field in topic_info.record_schema.field_list: self._dh_field_names.append(field.name) self._dh_field_types.append(field.type.value) @@ -127,9 +140,9 @@ def __init__(self, self._read_cnt = 32 if len(self._dh_fea_ids) > 1: - self._filter_fea_func = lambda record: ''.join( - [record.values[x] - for x in self._dh_fea_ids]).split(chr(2))[1] == '-1024' + self._filter_fea_func = ( + lambda record: ''.join([record.values[x] for x in self._dh_fea_ids] + ).split(chr(2))[1] == '-1024') else: dh_fea_id = self._dh_fea_ids[0] self._filter_fea_func = lambda record: record.values[dh_fea_id].split( @@ -244,16 +257,24 @@ def _datahub_generator(self): tid = 0 if shard_id not in self._offset_dict: - cursor_result = self._datahub.get_cursor(self._datahub_config.project, - self._datahub_config.topic, - shard_id, CursorType.OLDEST) + cursor_result = self._datahub.get_cursor( + self._datahub_config.project, + self._datahub_config.topic, + shard_id, + CursorType.OLDEST, + ) cursor = cursor_result.cursor else: cursor = self._offset_dict[shard_id] get_result = self._datahub.get_tuple_records( - self._datahub_config.project, self._datahub_config.topic, shard_id, - record_schema, cursor, self._read_cnt) + self._datahub_config.project, + self._datahub_config.topic, + shard_id, + record_schema, + cursor, + self._read_cnt, + ) count = get_result.record_count if count == 0: continue @@ -299,7 +320,8 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True) + reshuffle_each_iteration=True, + ) dataset = dataset.batch(self._data_config.batch_size) @@ -310,7 +332,8 @@ def _build(self, mode, params): # so that they could be feed into FeatureColumns dataset = dataset.map( map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls) + num_parallel_calls=self._data_config.num_parallel_calls, + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) if mode != tf.estimator.ModeKeys.PREDICT: dataset = dataset.map(lambda x: diff --git a/easy_rec/python/input/dummy_input.py b/easy_rec/python/input/dummy_input.py index f556a3686..6863b2210 100644 --- a/easy_rec/python/input/dummy_input.py +++ b/easy_rec/python/input/dummy_input.py @@ -16,18 +16,26 @@ class DummyInput(Input): Dummy Input is used to debug the performance bottleneck of data pipeline. """ - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - input_vals={}): - super(DummyInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + input_vals={}, + ): + super(DummyInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) self._input_vals = input_vals def _build(self, mode, params): diff --git a/easy_rec/python/input/hive_input.py b/easy_rec/python/input/hive_input.py index f5c8735af..b5f2ba7e0 100644 --- a/easy_rec/python/input/hive_input.py +++ b/easy_rec/python/input/hive_input.py @@ -11,17 +11,25 @@ class HiveInput(Input): """Common IO based interface, could run at local or on data science.""" - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(HiveInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(HiveInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) if input_path is None: return self._data_config = data_config @@ -32,8 +40,10 @@ def __init__(self, data_config=self._data_config, hive_config=self._hive_config) self._input_hdfs_path = hive_util.get_table_location( self._hive_config.table_name) - self._input_table_col_names, self._input_table_col_types = hive_util.get_all_cols( - self._hive_config.table_name) + ( + self._input_table_col_names, + self._input_table_col_types, + ) = hive_util.get_all_cols(self._hive_config.table_name) def _parse_csv(self, line): record_defaults = [] @@ -50,12 +60,15 @@ def _parse_csv(self, line): line, field_delim=self._data_config.separator, record_defaults=record_defaults, - name='decode_csv') + name='decode_csv', + ) fields = [] for x in self._input_fields: assert x in self._input_table_col_names, 'Column %s not in Table %s.' % ( - x, self._hive_config.table_name) + x, + self._hive_config.table_name, + ) fields.append(tmp_fields[self._input_table_col_names.index(x)]) # filter only valid fields @@ -88,7 +101,8 @@ def _build(self, mode, params): dataset = dataset.interleave( lambda x: tf.data.TextLineDataset(x), cycle_length=parallel_num, - num_parallel_calls=parallel_num) + num_parallel_calls=parallel_num, + ) if not self._data_config.file_shard: dataset = self._safe_shard(dataset) @@ -97,7 +111,8 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True) + reshuffle_each_iteration=True, + ) dataset = dataset.repeat(self.num_epochs) else: logging.info('eval files[%d]: %s' % diff --git a/easy_rec/python/input/hive_parquet_input.py b/easy_rec/python/input/hive_parquet_input.py index 7bb42dc42..e771c5237 100644 --- a/easy_rec/python/input/hive_parquet_input.py +++ b/easy_rec/python/input/hive_parquet_input.py @@ -14,17 +14,25 @@ class HiveParquetInput(Input): """Common IO based interface, could run at local or on data science.""" - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(HiveParquetInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(HiveParquetInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) if input_path is None: return self._data_config = data_config @@ -34,13 +42,17 @@ def __init__(self, hive_util = HiveUtils( data_config=self._data_config, hive_config=self._hive_config) input_hdfs_path = hive_util.get_table_location(self._hive_config.table_name) - self._input_table_col_names, self._input_table_col_types = hive_util.get_all_cols( - self._hive_config.table_name) + ( + self._input_table_col_names, + self._input_table_col_types, + ) = hive_util.get_all_cols(self._hive_config.table_name) self._all_hdfs_path = tf.gfile.Glob(os.path.join(input_hdfs_path, '*')) for x in self._input_fields: assert x in self._input_table_col_names, 'Column %s not in Table %s.' % ( - x, self._hive_config.table_name) + x, + self._hive_config.table_name, + ) self._record_defaults = [ self.get_type_defaults(t, v) @@ -115,7 +127,8 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True) + reshuffle_each_iteration=True, + ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) @@ -128,7 +141,8 @@ def _build(self, mode, params): # so that they could be feed into FeatureColumns dataset = dataset.map( map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls) + num_parallel_calls=self._data_config.num_parallel_calls, + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) diff --git a/easy_rec/python/input/hive_rtp_input.py b/easy_rec/python/input/hive_rtp_input.py index b7bbf2148..00b6971d8 100644 --- a/easy_rec/python/input/hive_rtp_input.py +++ b/easy_rec/python/input/hive_rtp_input.py @@ -13,17 +13,25 @@ class HiveRTPInput(Input): """Common IO based interface, could run at local or on data science.""" - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(HiveRTPInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(HiveRTPInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) if input_path is None: return self._data_config = data_config @@ -37,15 +45,18 @@ def __init__(self, if not isinstance(self._rtp_separator, str): self._rtp_separator = self._rtp_separator.encode('utf-8') logging.info('rtp separator = %s' % self._rtp_separator) - self._selected_cols = [c.strip() for c in self._data_config.selected_cols.split(',')] \ - if self._data_config.selected_cols else None + self._selected_cols = ([ + c.strip() for c in self._data_config.selected_cols.split(',') + ] if self._data_config.selected_cols else None) logging.info('select cols: %s' % self._selected_cols) hive_util = HiveUtils( data_config=self._data_config, hive_config=self._hive_config) self._input_hdfs_path = hive_util.get_table_location( self._hive_config.table_name) - self._input_table_col_names, self._input_table_col_types = hive_util.get_all_cols( - self._hive_config.table_name) + ( + self._input_table_col_names, + self._input_table_col_types, + ) = hive_util.get_all_cols(self._hive_config.table_name) def _parse_csv(self, line): non_feature_cols = self._label_fields @@ -65,7 +76,8 @@ def _parse_csv(self, line): line, field_delim=self._rtp_separator, record_defaults=record_defaults, - name='decode_csv') + name='decode_csv', + ) print('tmp_fields: ', tmp_fields) fields = [] @@ -83,13 +95,14 @@ def _parse_csv(self, line): ] feature_num = len(record_types) - check_list = [ + check_list = ([ tf.py_func( check_split, [fields[-1], self._data_config.separator, len(record_types)], - Tout=tf.bool) - ] if self._check_mode else [] + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): fields = tf.string_split( fields[-1], self._data_config.separator, skip_empty=False) @@ -139,7 +152,8 @@ def _build(self, mode, params): dataset = dataset.interleave( lambda x: tf.data.TextLineDataset(x), cycle_length=parallel_num, - num_parallel_calls=parallel_num) + num_parallel_calls=parallel_num, + ) if not self._data_config.file_shard: dataset = self._safe_shard(dataset) @@ -148,7 +162,8 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True) + reshuffle_each_iteration=True, + ) dataset = dataset.repeat(self.num_epochs) else: logging.info('eval files[%d]: %s' % diff --git a/easy_rec/python/input/input.py b/easy_rec/python/input/input.py index f53c4ee45..227ee861a 100644 --- a/easy_rec/python/input/input.py +++ b/easy_rec/python/input/input.py @@ -18,14 +18,17 @@ from easy_rec.python.utils import conditional from easy_rec.python.utils import config_util from easy_rec.python.utils import constant -from easy_rec.python.utils.check_utils import check_split -from easy_rec.python.utils.check_utils import check_string_to_number from easy_rec.python.utils.expr_util import get_expression from easy_rec.python.utils.input_utils import get_type_defaults -from easy_rec.python.utils.load_class import get_register_class_meta -from easy_rec.python.utils.load_class import load_by_path from easy_rec.python.utils.tf_utils import get_tf_type +from easy_rec.python.utils.check_utils import ( # NOQA + check_split, check_string_to_number, +) +from easy_rec.python.utils.load_class import ( # NOQA + get_register_class_meta, load_by_path, +) + if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -34,18 +37,19 @@ class Input(six.with_metaclass(_meta_type, object)): - DATA_OFFSET = 'DATA_OFFSET' - def __init__(self, - data_config, - feature_configs, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - **kwargs): + def __init__( + self, + data_config, + feature_configs, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + **kwargs, + ): self._pipeline_config = pipeline_config self._data_config = data_config self._check_mode = check_mode @@ -282,8 +286,10 @@ def create_multi_placeholders(self, export_config): else: placeholder_name = 'input_%d' % fid if input_name in export_fields_name: - tf_type = self._multi_value_types[input_name] if input_name in self._multi_value_types \ - else get_tf_type(self._input_field_types[fid]) + tf_type = ( + self._multi_value_types[input_name] + if input_name in self._multi_value_types else get_tf_type( + self._input_field_types[fid])) logging.info('multi value input_name: %s, dtype: %s' % (input_name, tf_type)) finput = array_ops.placeholder( @@ -334,7 +340,8 @@ def create_placeholders(self, export_config): features[input_name] = tf.string_to_number( input_vals[:, tmp_id], tf_type, - name='input_str_to_%s' % tf_type.name) + name='input_str_to_%s' % tf_type.name, + ) else: if ftype not in [DatasetConfig.STRING]: logging.warning('unexpected field type: ftype=%s tf_type=%s' % @@ -348,10 +355,11 @@ def _get_features(self, fields): def _get_labels(self, fields): labels = fields['label'] - return OrderedDict([ - (x, tf.squeeze(labels[x], axis=1) if len(labels[x].get_shape()) == 2 and - labels[x].get_shape()[1] == 1 else labels[x]) for x in labels - ]) + return OrderedDict([( + x, + (tf.squeeze(labels[x], axis=1) if len(labels[x].get_shape()) == 2 and + labels[x].get_shape()[1] == 1 else labels[x]), + ) for x in labels]) def _as_string(self, field, fc): if field.dtype == tf.string: @@ -359,11 +367,12 @@ def _as_string(self, field, fc): if field.dtype in [tf.float32, tf.double]: feature_name = fc.feature_name if fc.HasField( 'feature_name') else fc.input_names[0] - assert fc.precision > 0, 'fc.precision not set for feature[%s], it is dangerous to convert ' \ - 'float or double to string due to precision problem, it is suggested ' \ - ' to convert them into string format before using EasyRec; ' \ - 'if you really need to do so, please set precision (the number of ' \ - 'decimal digits) carefully.' % feature_name + assert fc.precision > 0, ( + 'fc.precision not set for feature[%s], it is dangerous to convert ' + 'float or double to string due to precision problem, it is suggested ' + ' to convert them into string format before using EasyRec; ' + 'if you really need to do so, please set precision (the number of ' + 'decimal digits) carefully.' % feature_name) precision = None if field.dtype in [tf.float32, tf.double]: if fc.precision > 0: @@ -381,9 +390,12 @@ def _parse_combo_feature(self, fc, parsed_dict, field_dict): 'feature_name') else fc.input_names[0] if len(fc.combo_input_seps) > 0: - assert len(fc.combo_input_seps) == len(fc.input_names), \ - 'len(combo_separator)[%d] != len(fc.input_names)[%d]' % ( - len(fc.combo_input_seps), len(fc.input_names)) + assert len(fc.combo_input_seps) == len( + fc.input_names + ), 'len(combo_separator)[%d] != len(fc.input_names)[%d]' % ( + len(fc.combo_input_seps), + len(fc.input_names), + ) def _get_input_sep(input_id): if input_id < len(fc.combo_input_seps): @@ -399,9 +411,11 @@ def _get_input_sep(input_id): key = feature_name input_sep = _get_input_sep(input_id) if input_sep != '': - assert field_dict[ - input_name].dtype == tf.string, 'could not apply string_split to input-name[%s] dtype=%s' % ( - input_name, field_dict[input_name].dtype) + assert field_dict[input_name].dtype == tf.string, ( + 'could not apply string_split to input-name[%s] dtype=%s' % ( + input_name, + field_dict[input_name].dtype, + )) parsed_dict[key] = tf.string_split(field_dict[input_name], input_sep) else: parsed_dict[key] = self._as_string(field_dict[input_name], fc) @@ -411,9 +425,11 @@ def _get_input_sep(input_id): for input_id, input_name in enumerate(fc.input_names): input_sep = fc.combo_input_seps[input_id] if len(input_sep) > 0: - assert field_dict[ - input_name].dtype == tf.string, 'could not apply string_split to input-name[%s] dtype=%s' % ( - input_name, field_dict[input_name].dtype) + assert field_dict[input_name].dtype == tf.string, ( + 'could not apply string_split to input-name[%s] dtype=%s' % ( + input_name, + field_dict[input_name].dtype, + )) split_inputs.append( tf.string_split(field_dict[input_name], fc.combo_input_seps[input_id])) @@ -441,8 +457,10 @@ def _parse_tag_feature(self, fc, parsed_dict, field_dict): elif len(field.get_shape()) == 2: field = tf.squeeze(field, axis=-1) if fc.HasField('kv_separator') and len(fc.input_names) > 1: - assert False, 'Tag Feature Error, ' \ - 'Cannot set kv_separator and multi input_names in one feature config. Feature: %s.' % input_0 + assert False, ( + 'Tag Feature Error, ' + 'Cannot set kv_separator and multi input_names in one feature config. Feature: %s.' + % input_0) parsed_dict[feature_name] = tf.string_split(field, fc.separator) if fc.HasField('kv_separator'): indices = parsed_dict[feature_name].indices @@ -462,38 +480,46 @@ def _parse_tag_feature(self, fc, parsed_dict, field_dict): parsed_dict[feature_name + '_w'] = tf.sparse.SparseTensor( indices, tmp_vs, parsed_dict[feature_name].dense_shape) if not fc.HasField('hash_bucket_size') and fc.num_buckets > 0: - check_list = [ + check_list = ([ tf.py_func( check_string_to_number, [parsed_dict[feature_name].values, input_0], - Tout=tf.bool) - ] if self._check_mode else [] + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): vals = tf.string_to_number( parsed_dict[feature_name].values, tf.int32, - name='tag_fea_%s' % input_0) + name='tag_fea_%s' % input_0, + ) parsed_dict[feature_name] = tf.sparse.SparseTensor( - parsed_dict[feature_name].indices, vals, - parsed_dict[feature_name].dense_shape) + parsed_dict[feature_name].indices, + vals, + parsed_dict[feature_name].dense_shape, + ) if len(fc.input_names) > 1: input_1 = fc.input_names[1] field = field_dict[input_1] if len(field.get_shape()) == 0: field = tf.expand_dims(field, axis=0) field = tf.string_split(field, fc.separator) - check_list = [ + check_list = ([ tf.py_func( - check_string_to_number, [field.values, input_1], Tout=tf.bool) - ] if self._check_mode else [] + check_string_to_number, + [field.values, input_1], + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): field_vals = tf.string_to_number( field.values, tf.float32, name='tag_wgt_str_2_flt_%s' % input_1) assert_op = tf.assert_equal( tf.shape(field_vals)[0], tf.shape(parsed_dict[feature_name].values)[0], - message='TagFeature Error: The size of %s not equal to the size of %s. Please check input: %s and %s.' - % (input_0, input_1, input_0, input_1)) + message='TagFeature Error: The size of %s not equal to %s. Please check input: %s and %s.' + % (input_0, input_1, input_0, input_1), + ) with tf.control_dependencies([assert_op]): field = tf.sparse.SparseTensor(field.indices, tf.identity(field_vals), field.dense_shape) @@ -510,18 +536,24 @@ def _parse_expr_feature(self, fc, parsed_dict, field_dict): for input_name in fc.input_names: new_input_name = prefix + input_name if field_dict[input_name].dtype == tf.string: - check_list = [ + check_list = ([ tf.py_func( - check_string_to_number, [field_dict[input_name], input_name], - Tout=tf.bool) - ] if self._check_mode else [] + check_string_to_number, + [field_dict[input_name], input_name], + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): parsed_dict[new_input_name] = tf.string_to_number( field_dict[input_name], tf.float64, - name='%s_str_2_int_for_expr' % new_input_name) + name='%s_str_2_int_for_expr' % new_input_name, + ) elif field_dict[input_name].dtype in [ - tf.int32, tf.int64, tf.double, tf.float32 + tf.int32, + tf.int64, + tf.double, + tf.float32, ]: parsed_dict[new_input_name] = tf.cast(field_dict[input_name], tf.float64) @@ -543,16 +575,19 @@ def _parse_id_feature(self, fc, parsed_dict, field_dict): parsed_dict[feature_name] = self._as_string(field_dict[input_0], fc) elif fc.num_buckets > 0: if parsed_dict[feature_name].dtype == tf.string: - check_list = [ + check_list = ([ tf.py_func( - check_string_to_number, [parsed_dict[feature_name], input_0], - Tout=tf.bool) - ] if self._check_mode else [] + check_string_to_number, + [parsed_dict[feature_name], input_0], + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): parsed_dict[feature_name] = tf.string_to_number( parsed_dict[feature_name], tf.int32, - name='%s_str_2_int' % input_0) + name='%s_str_2_int' % input_0, + ) def _parse_raw_feature(self, fc, parsed_dict, field_dict): input_0 = fc.input_names[0] @@ -566,22 +601,28 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): vals = field_dict[input_0] segment_ids = tf.range(0, tf.shape(vals)[0]) if fc.raw_input_dim > 1: - check_list = [ + check_list = ([ tf.py_func( - check_split, [vals, fc.separator, fc.raw_input_dim, input_0], - Tout=tf.bool) - ] if self._check_mode else [] + check_split, + [vals, fc.separator, fc.raw_input_dim, input_0], + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): tmp_fea = tf.string_split(vals, fc.separator) - check_list = [ + check_list = ([ tf.py_func( - check_string_to_number, [tmp_fea.values, input_0], Tout=tf.bool) - ] if self._check_mode else [] + check_string_to_number, + [tmp_fea.values, input_0], + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): tmp_vals = tf.string_to_number( tmp_fea.values, tf.float32, - name='multi_raw_fea_to_flt_%s' % input_0) + name='multi_raw_fea_to_flt_%s' % input_0, + ) if fc.HasField('seq_multi_sep') and fc.HasField('combiner'): emb = tf.reshape(tmp_vals, [-1, fc.raw_input_dim]) if fc.combiner == 'max': @@ -600,7 +641,8 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): tmp_fea.indices, [tf.shape(field_dict[input_0])[0], fc.raw_input_dim], tmp_vals, - default_value=0) + default_value=0, + ) elif fc.HasField('seq_multi_sep') and fc.HasField('combiner'): check_list = [ tf.py_func(check_string_to_number, [vals, input_0], Tout=tf.bool) @@ -620,11 +662,13 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): assert False, 'unsupported combine operator: ' + fc.combiner parsed_dict[feature_name] = emb else: - check_list = [ + check_list = ([ tf.py_func( - check_string_to_number, [field_dict[input_0], input_0], - Tout=tf.bool) - ] if self._check_mode else [] + check_string_to_number, + [field_dict[input_0], input_0], + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): parsed_dict[feature_name] = tf.string_to_number( field_dict[input_0], tf.float32) @@ -645,9 +689,8 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): parsed_dict[feature_name] = self._normalizer_fn[feature_name]( parsed_dict[feature_name]) - if not fc.boundaries and fc.num_buckets <= 1 and \ - fc.embedding_dim > 0 and \ - self._data_config.sample_weight != input_0: + if (not fc.boundaries and fc.num_buckets <= 1 and fc.embedding_dim > 0 and + self._data_config.sample_weight != input_0): # may need by wide model and deep model to project # raw values to a vector, it maybe better implemented # by a ProjectionColumn later @@ -666,11 +709,13 @@ def _parse_raw_feature(self, fc, parsed_dict, field_dict): parsed_dict[feature_name + '_raw_proj_id'] = tf.SparseTensor( indices=indices, values=indices_1[:, 0], - dense_shape=[sample_num, fc.raw_input_dim]) + dense_shape=[sample_num, fc.raw_input_dim], + ) parsed_dict[feature_name + '_raw_proj_val'] = tf.SparseTensor( indices=indices, values=tf.reshape(tmp_parsed, [-1]), - dense_shape=[sample_num, fc.raw_input_dim]) + dense_shape=[sample_num, fc.raw_input_dim], + ) # self._appended_fields.append(input_0 + '_raw_proj_id') # self._appended_fields.append(input_0 + '_raw_proj_val') @@ -693,58 +738,67 @@ def _parse_seq_feature(self, fc, parsed_dict, field_dict): # 3 dimensional sparse tensor out_shape = tf.concat( [parsed_dict[feature_name].dense_shape, multi_vals.dense_shape[1:]], - axis=0) + axis=0, + ) parsed_dict[feature_name] = tf.sparse.SparseTensor( out_indices, multi_vals.values, out_shape) - if (fc.num_buckets > 1 and fc.max_val == fc.min_val): - check_list = [ + if fc.num_buckets > 1 and fc.max_val == fc.min_val: + check_list = ([ tf.py_func( check_string_to_number, [parsed_dict[feature_name].values, input_0], - Tout=tf.bool) - ] if self._check_mode else [] + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): parsed_dict[feature_name] = tf.sparse.SparseTensor( parsed_dict[feature_name].indices, tf.string_to_number( parsed_dict[feature_name].values, tf.int64, - name='sequence_str_2_int_%s' % input_0), - parsed_dict[feature_name].dense_shape) + name='sequence_str_2_int_%s' % input_0, + ), + parsed_dict[feature_name].dense_shape, + ) elif sub_feature_type == fc.RawFeature: - check_list = [ + check_list = ([ tf.py_func( check_string_to_number, [parsed_dict[feature_name].values, input_0], - Tout=tf.bool) - ] if self._check_mode else [] + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): parsed_dict[feature_name] = tf.sparse.SparseTensor( parsed_dict[feature_name].indices, tf.string_to_number( parsed_dict[feature_name].values, tf.float32, - name='sequence_str_2_float_%s' % input_0), - parsed_dict[feature_name].dense_shape) + name='sequence_str_2_float_%s' % input_0, + ), + parsed_dict[feature_name].dense_shape, + ) if fc.num_buckets > 1 and fc.max_val > fc.min_val: normalized_values = (parsed_dict[feature_name].values - fc.min_val) / ( fc.max_val - fc.min_val) parsed_dict[feature_name] = tf.sparse.SparseTensor( - parsed_dict[feature_name].indices, normalized_values, - parsed_dict[feature_name].dense_shape) + parsed_dict[feature_name].indices, + normalized_values, + parsed_dict[feature_name].dense_shape, + ) else: parsed_dict[feature_name] = field - if not fc.boundaries and fc.num_buckets <= 1 and\ - self._data_config.sample_weight != input_0 and\ - sub_feature_type == fc.RawFeature and\ - fc.raw_input_dim == 1: - logging.info( - 'Not set boundaries or num_buckets or hash_bucket_size, %s will process as two dimension sequence raw feature' - % feature_name) + if (not fc.boundaries and fc.num_buckets <= 1 and + self._data_config.sample_weight != input_0 and + sub_feature_type == fc.RawFeature and fc.raw_input_dim == 1): + logging.info(('Not set boundaries or num_buckets or hash_bucket_size, ' + '%s will process as two dimension sequence raw feature') % + feature_name) parsed_dict[feature_name] = tf.sparse_to_dense( parsed_dict[feature_name].indices, [tf.shape(parsed_dict[feature_name])[0], fc.sequence_length], - parsed_dict[feature_name].values) + parsed_dict[feature_name].values, + ) sample_num = tf.to_int64(tf.shape(parsed_dict[feature_name])[0]) indices_0 = tf.range(sample_num, dtype=tf.int64) indices_1 = tf.range(fc.sequence_length, dtype=tf.int64) @@ -759,11 +813,13 @@ def _parse_seq_feature(self, fc, parsed_dict, field_dict): parsed_dict[feature_name + '_raw_proj_id'] = tf.SparseTensor( indices=indices, values=indices_1[:, 0], - dense_shape=[sample_num, fc.sequence_length]) + dense_shape=[sample_num, fc.sequence_length], + ) parsed_dict[feature_name + '_raw_proj_val'] = tf.SparseTensor( indices=indices, values=tf.reshape(tmp_parsed, [-1]), - dense_shape=[sample_num, fc.sequence_length]) + dense_shape=[sample_num, fc.sequence_length], + ) elif (not fc.boundaries and fc.num_buckets <= 1 and self._data_config.sample_weight != input_0 and sub_feature_type == fc.RawFeature and fc.raw_input_dim > 1): @@ -772,10 +828,14 @@ def _parse_seq_feature(self, fc, parsed_dict, field_dict): ' %s will process as three dimension sequence raw feature' % feature_name) parsed_dict[feature_name] = tf.sparse_to_dense( - parsed_dict[feature_name].indices, [ - tf.shape(parsed_dict[feature_name])[0], fc.sequence_length, - fc.raw_input_dim - ], parsed_dict[feature_name].values) + parsed_dict[feature_name].indices, + [ + tf.shape(parsed_dict[feature_name])[0], + fc.sequence_length, + fc.raw_input_dim, + ], + parsed_dict[feature_name].values, + ) sample_num = tf.to_int64(tf.shape(parsed_dict[feature_name])[0]) indices_0 = tf.range(sample_num, dtype=tf.int64) indices_1 = tf.range(fc.sequence_length, dtype=tf.int64) @@ -795,11 +855,13 @@ def _parse_seq_feature(self, fc, parsed_dict, field_dict): parsed_dict[feature_name + '_raw_proj_id'] = tf.SparseTensor( indices=indices, values=indices_1[:, 0], - dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim]) + dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim], + ) parsed_dict[feature_name + '_raw_proj_val'] = tf.SparseTensor( indices=indices, values=tf.reshape(parsed_dict[feature_name], [-1]), - dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim]) + dense_shape=[sample_num, fc.sequence_length, fc.raw_input_dim], + ) # self._appended_fields.append(input_0 + '_raw_proj_id') # self._appended_fields.append(input_0 + '_raw_proj_val') @@ -892,14 +954,18 @@ def _preprocess(self, field_dict): if field_dict[input_name].dtype == tf.string: if self._label_dim[input_id] > 1: logging.info('will split labels[%d]=%s' % (input_id, input_name)) - check_list = [ + check_list = ([ tf.py_func( - check_split, [ - field_dict[input_name], self._label_sep[input_id], - self._label_dim[input_id], input_name + check_split, + [ + field_dict[input_name], + self._label_sep[input_id], + self._label_dim[input_id], + input_name, ], - Tout=tf.bool) - ] if self._check_mode else [] + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): label_dict[input_name] = tf.string_split( field_dict[input_name], self._label_sep[input_id]).values @@ -907,17 +973,22 @@ def _preprocess(self, field_dict): [-1, self._label_dim[input_id]]) else: label_dict[input_name] = field_dict[input_name] - check_list = [ + check_list = ([ tf.py_func( - check_string_to_number, [label_dict[input_name], input_name], - Tout=tf.bool) - ] if self._check_mode else [] + check_string_to_number, + [label_dict[input_name], input_name], + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): label_dict[input_name] = tf.string_to_number( label_dict[input_name], tf.float32, name=input_name) else: assert field_dict[input_name].dtype in [ - tf.float32, tf.double, tf.int32, tf.int64 + tf.float32, + tf.double, + tf.int32, + tf.int64, ], 'invalid label dtype: %s' % str(field_dict[input_name].dtype) label_dict[input_name] = field_dict[input_name] @@ -981,7 +1052,7 @@ def _lookup(args, pad=True): indices_1 = tf.range(0, n, dtype=tf.int64) indices = [ tf.expand_dims(indices_0, axis=1), - tf.expand_dims(indices_1, axis=1) + tf.expand_dims(indices_1, axis=1), ] indices = tf.concat(indices, axis=1) return tf.sparse.SparseTensor(indices, vals, [1, n]) @@ -998,7 +1069,8 @@ def _lookup(args, pad=True): indices = tf.concat( [tf.expand_dims(indices_0, axis=1), tf.expand_dims(indices_1, axis=1)], - axis=1) + axis=1, + ) shapes = tf.stack([batch_size, tf.reduce_max(indices_1) + 1]) return tf.sparse.SparseTensor(indices, vals, shapes) @@ -1040,8 +1112,11 @@ def _input_fn(mode=None, params=None, config=None): tf.estimator.export.ServingInputReceiver instance """ self._pre_build(mode, params) - if mode in (tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL, - tf.estimator.ModeKeys.PREDICT): + if mode in ( + tf.estimator.ModeKeys.TRAIN, + tf.estimator.ModeKeys.EVAL, + tf.estimator.ModeKeys.PREDICT, + ): # build dataset from self._config.input_path self._mode = mode dataset = self._build(mode, params) diff --git a/easy_rec/python/input/kafka_dataset.py b/easy_rec/python/input/kafka_dataset.py index 22ae45b90..20ed51872 100644 --- a/easy_rec/python/input/kafka_dataset.py +++ b/easy_rec/python/input/kafka_dataset.py @@ -31,16 +31,18 @@ class KafkaDataset(dataset_ops.Dataset): """A Kafka Dataset that consumes the message.""" - def __init__(self, - topics, - servers='localhost', - group='', - eof=False, - timeout=1000, - config_global=None, - config_topic=None, - message_key=False, - message_offset=False): + def __init__( + self, + topics, + servers='localhost', + group='', + eof=False, + timeout=1000, + config_global=None, + config_topic=None, + message_key=False, + message_offset=False, + ): """Create a KafkaReader. Args: @@ -107,24 +109,27 @@ def output_classes(self): return (ops.Tensor, ops.Tensor) elif self._message_key and self._message_offset: return (ops.Tensor, ops.Tensor, ops.Tensor) - return (ops.Tensor) + return ops.Tensor @property def output_shapes(self): if self._message_key ^ self._message_offset: - return ((tensor_shape.TensorShape([]), tensor_shape.TensorShape([]))) + return (tensor_shape.TensorShape([]), tensor_shape.TensorShape([])) elif self._message_key and self._message_offset: - return ((tensor_shape.TensorShape([]), tensor_shape.TensorShape([]), - tensor_shape.TensorShape([]))) - return ((tensor_shape.TensorShape([]))) + return ( + tensor_shape.TensorShape([]), + tensor_shape.TensorShape([]), + tensor_shape.TensorShape([]), + ) + return tensor_shape.TensorShape([]) @property def output_types(self): if self._message_key ^ self._message_offset: - return ((dtypes.string, dtypes.string)) + return (dtypes.string, dtypes.string) elif self._message_key and self._message_offset: - return ((dtypes.string, dtypes.string, dtypes.string)) - return ((dtypes.string)) + return (dtypes.string, dtypes.string, dtypes.string) + return dtypes.string def write_kafka_v2(message, topic, servers='localhost', name=None): diff --git a/easy_rec/python/input/odps_input.py b/easy_rec/python/input/odps_input.py index bbd08bc39..c3068deff 100644 --- a/easy_rec/python/input/odps_input.py +++ b/easy_rec/python/input/odps_input.py @@ -13,36 +13,45 @@ class OdpsInput(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(OdpsInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(OdpsInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) def _build(self, mode, params): # check data_config are consistent with odps tables odps_util.check_input_field_and_types(self._data_config) selected_cols = ','.join(self._input_fields) - if self._data_config.chief_redundant and \ - mode == tf.estimator.ModeKeys.TRAIN: + if self._data_config.chief_redundant and mode == tf.estimator.ModeKeys.TRAIN: reader = tf.TableRecordReader( csv_delimiter=self._data_config.separator, selected_cols=selected_cols, slice_count=max(self._task_num - 1, 1), - slice_id=max(self._task_index - 1, 0)) + slice_id=max(self._task_index - 1, 0), + ) else: reader = tf.TableRecordReader( csv_delimiter=self._data_config.separator, selected_cols=selected_cols, slice_count=self._task_num, - slice_id=self._task_index) + slice_id=self._task_index, + ) if type(self._input_path) != list: self._input_path = self._input_path.split(',') @@ -55,7 +64,8 @@ def _build(self, mode, params): self._input_path, num_epochs=self.num_epochs, shuffle=self._data_config.shuffle, - num_slices=self._data_config.pai_worker_slice_num * self._task_num) + num_slices=self._data_config.pai_worker_slice_num * self._task_num, + ) work_queue.add_summary() file_queue = work_queue.input_producer() reader = tf.TableRecordReader() @@ -64,7 +74,8 @@ def _build(self, mode, params): self._input_path, num_epochs=self.num_epochs, capacity=1000, - shuffle=self._data_config.shuffle) + shuffle=self._data_config.shuffle, + ) else: file_queue = tf.train.string_input_producer( self._input_path, num_epochs=1, capacity=1000, shuffle=False) @@ -78,7 +89,8 @@ def _build(self, mode, params): value, record_defaults=record_defaults, field_delim=self._data_config.separator, - name='decode_csv') + name='decode_csv', + ) inputs = {self._input_fields[x]: fields[x] for x in self._effective_fids} for x in self._label_fids: diff --git a/easy_rec/python/input/odps_input_v2.py b/easy_rec/python/input/odps_input_v2.py index bd58de390..9cba3d0e2 100644 --- a/easy_rec/python/input/odps_input_v2.py +++ b/easy_rec/python/input/odps_input_v2.py @@ -15,17 +15,25 @@ class OdpsInputV2(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(OdpsInputV2, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(OdpsInputV2, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) def _parse_table(self, *fields): fields = list(fields) @@ -48,42 +56,44 @@ def _build(self, mode, params): for x, v in zip(self._input_field_types, self._input_field_defaults) ] - if self._data_config.pai_worker_queue and \ - mode == tf.estimator.ModeKeys.TRAIN: + if self._data_config.pai_worker_queue and mode == tf.estimator.ModeKeys.TRAIN: logging.info('pai_worker_slice_num = %d' % self._data_config.pai_worker_slice_num) work_queue = pai.data.WorkQueue( self._input_path, num_epochs=self.num_epochs, shuffle=self._data_config.shuffle, - num_slices=self._data_config.pai_worker_slice_num * self._task_num) + num_slices=self._data_config.pai_worker_slice_num * self._task_num, + ) que_paths = work_queue.input_dataset() dataset = tf.data.TableRecordDataset( que_paths, record_defaults=record_defaults, selected_cols=selected_cols) - elif self._data_config.chief_redundant and \ - mode == tf.estimator.ModeKeys.TRAIN: + elif self._data_config.chief_redundant and mode == tf.estimator.ModeKeys.TRAIN: dataset = tf.data.TableRecordDataset( self._input_path, record_defaults=record_defaults, selected_cols=selected_cols, slice_id=max(self._task_index - 1, 0), - slice_count=max(self._task_num - 1, 1)) + slice_count=max(self._task_num - 1, 1), + ) else: dataset = tf.data.TableRecordDataset( self._input_path, record_defaults=record_defaults, selected_cols=selected_cols, slice_id=self._task_index, - slice_count=self._task_num) + slice_count=self._task_num, + ) if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.shuffle: dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True) + reshuffle_each_iteration=True, + ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) @@ -98,7 +108,8 @@ def _build(self, mode, params): # so that they could be feed into FeatureColumns dataset = dataset.map( map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls) + num_parallel_calls=self._data_config.num_parallel_calls, + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) diff --git a/easy_rec/python/input/odps_input_v3.py b/easy_rec/python/input/odps_input_v3.py index 6ec737ca1..ba3f5389c 100644 --- a/easy_rec/python/input/odps_input_v3.py +++ b/easy_rec/python/input/odps_input_v3.py @@ -19,22 +19,30 @@ class OdpsInputV3(Input): """Common IO based interface, could run at local or on data science.""" - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(OdpsInputV3, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(OdpsInputV3, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) self._num_epoch = 0 if common_io is None: - logging.error(''' + logging.error(""" please install common_io pip install - https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.4.2%2Btunnel-py2.py3-none-any.whl''' + https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.4.2%2Btunnel-py2.py3-none-any.whl""" ) sys.exit(1) @@ -67,7 +75,8 @@ def _odps_read(self): table_path, selected_cols=selected_cols, slice_id=self._task_index, - slice_count=self._task_num) + slice_count=self._task_num, + ) total_records_num = reader.get_row_count() batch_num = int(total_records_num / self._data_config.batch_size) res_num = total_records_num - batch_num * self._data_config.batch_size @@ -107,7 +116,8 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True) + reshuffle_each_iteration=True, + ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) @@ -120,7 +130,8 @@ def _build(self, mode, params): # so that they could be feed into FeatureColumns dataset = dataset.map( map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls) + num_parallel_calls=self._data_config.num_parallel_calls, + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) diff --git a/easy_rec/python/input/odps_rtp_input.py b/easy_rec/python/input/odps_rtp_input.py index 6ae6096e0..bcf6d0a7e 100644 --- a/easy_rec/python/input/odps_rtp_input.py +++ b/easy_rec/python/input/odps_rtp_input.py @@ -29,17 +29,25 @@ class OdpsRTPInput(Input): selected columns are labels """ - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(OdpsRTPInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(OdpsRTPInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) logging.info('input_fields: %s label_fields: %s' % (','.join(self._input_fields), ','.join(self._label_fields))) @@ -47,8 +55,7 @@ def _parse_table(self, *fields): fields = list(fields) labels = fields[:-1] - selected_cols = self._data_config.selected_cols \ - if self._data_config.selected_cols else None + selected_cols = self._data_config.selected_cols if self._data_config.selected_cols else None non_feature_cols = self._label_fields if selected_cols: cols = [c.strip() for c in selected_cols.split(',')] @@ -72,13 +79,14 @@ def _parse_table(self, *fields): logging.info('field_delim = %s, input_field_name = %d' % (self._data_config.separator, len(record_types))) - check_list = [ + check_list = ([ tf.py_func( check_split, [fields[-1], self._data_config.separator, len(record_types)], - Tout=tf.bool) - ] if self._check_mode else [] + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): fields = str_split_by_chr( fields[-1], self._data_config.separator, skip_empty=False) @@ -105,43 +113,45 @@ def _build(self, mode, params): assert len( self._input_path) > 0, 'match no files with %s' % self._input_path - selected_cols = self._data_config.selected_cols \ - if self._data_config.selected_cols else None + selected_cols = self._data_config.selected_cols if self._data_config.selected_cols else None if selected_cols: cols = [c.strip() for c in selected_cols.split(',')] record_defaults = [ - self.get_type_defaults(t, v) - for x, t, v in zip(self._input_fields, self._input_field_types, - self._input_field_defaults) - if x in cols[:-1] + self.get_type_defaults(t, v) for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) if x in cols[:-1] ] print('selected_cols: %s; defaults num: %d' % (','.join(cols), len(record_defaults))) else: record_defaults = [ - self.get_type_defaults(t, v) - for x, t, v in zip(self._input_fields, self._input_field_types, - self._input_field_defaults) - if x in self._label_fields + self.get_type_defaults(t, v) for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) if x in self._label_fields ] # the actual features are in one single column record_defaults.append( self._data_config.separator.join([ - str(self.get_type_defaults(t, v)) - for x, t, v in zip(self._input_fields, self._input_field_types, - self._input_field_defaults) - if x not in self._label_fields + str(self.get_type_defaults(t, v)) for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) if x not in self._label_fields ])) - if self._data_config.pai_worker_queue and \ - mode == tf.estimator.ModeKeys.TRAIN: + if self._data_config.pai_worker_queue and mode == tf.estimator.ModeKeys.TRAIN: logging.info('pai_worker_slice_num = %d' % self._data_config.pai_worker_slice_num) work_queue = pai.data.WorkQueue( self._input_path, num_epochs=self.num_epochs, shuffle=self._data_config.shuffle, - num_slices=self._data_config.pai_worker_slice_num * self._task_num) + num_slices=self._data_config.pai_worker_slice_num * self._task_num, + ) que_paths = work_queue.input_dataset() dataset = tf.data.TableRecordDataset( que_paths, @@ -153,14 +163,16 @@ def _build(self, mode, params): record_defaults=record_defaults, selected_cols=selected_cols, slice_id=self._task_index, - slice_count=self._task_num) + slice_count=self._task_num, + ) if mode == tf.estimator.ModeKeys.TRAIN: if self._data_config.shuffle: dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True) + reshuffle_each_iteration=True, + ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) @@ -175,7 +187,8 @@ def _build(self, mode, params): # so that they could be feed into FeatureColumns dataset = dataset.map( map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls) + num_parallel_calls=self._data_config.num_parallel_calls, + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) diff --git a/easy_rec/python/input/odps_rtp_input_v2.py b/easy_rec/python/input/odps_rtp_input_v2.py index 77edb46c1..9516084a0 100644 --- a/easy_rec/python/input/odps_rtp_input_v2.py +++ b/easy_rec/python/input/odps_rtp_input_v2.py @@ -31,18 +31,26 @@ class OdpsRTPInputV2(OdpsRTPInput): selected columns are labels """ - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - fg_json_path=None, - pipeline_config=None): - super(OdpsRTPInputV2, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + fg_json_path=None, + pipeline_config=None, + ): + super(OdpsRTPInputV2, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) if fg_json_path.startswith('!'): fg_json_path = fg_json_path[1:] self._fg_config_path = fg_json_path diff --git a/easy_rec/python/input/parquet_input.py b/easy_rec/python/input/parquet_input.py index dcc6e867f..0d6342772 100644 --- a/easy_rec/python/input/parquet_input.py +++ b/easy_rec/python/input/parquet_input.py @@ -18,18 +18,27 @@ class ParquetInput(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - **kwargs): - super(ParquetInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config, **kwargs) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + **kwargs, + ): + super(ParquetInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + **kwargs, + ) self._need_pack = True if input_path is None: return @@ -108,7 +117,7 @@ def _rebuild_que(self): def _sample_generator(self): if not self._proc_start: self._proc_start = True - for proc in (self._proc_arr): + for proc in self._proc_arr: self._proc_start_que.put(True) logging.info('task[%s] data_proc=%s is_alive=%s' % (self._task_index, proc, proc.is_alive())) @@ -142,9 +151,12 @@ def _sample_generator(self): yield sample if fetch_good_cnt % 200 == 0: logging.info( - 'task[%d] fetch_batch_cnt=%d, fetch_timeout_cnt=%d, qsize=%d' % - (self._task_index, fetch_good_cnt, fetch_timeout_cnt, - self._data_que.qsize())) + 'task[%d] fetch_batch_cnt=%d, fetch_timeout_cnt=%d, qsize=%d' % ( + self._task_index, + fetch_good_cnt, + fetch_timeout_cnt, + self._data_que.qsize(), + )) except queue.Empty: fetch_timeout_cnt += 1 if done_proc_cnt >= len(self._proc_arr): @@ -203,13 +215,17 @@ def _to_fea_dict(self, input_dict): if len(self._sparse_fea_names) > 0: if self._has_ev: - tmp_vals, tmp_lens = input_dict['sparse_fea'][1], input_dict[ - 'sparse_fea'][0] + tmp_vals, tmp_lens = ( + input_dict['sparse_fea'][1], + input_dict['sparse_fea'][0], + ) fea_dict['sparse_fea'] = (tmp_vals, tmp_lens) else: - tmp_vals, tmp_lens = input_dict['sparse_fea'][1], input_dict[ - 'sparse_fea'][0] + tmp_vals, tmp_lens = ( + input_dict['sparse_fea'][1], + input_dict['sparse_fea'][0], + ) num_buckets = -1 for fc in self._feature_configs: if fc.num_buckets > 0: @@ -217,7 +233,9 @@ def _to_fea_dict(self, input_dict): num_buckets = fc.num_buckets else: assert num_buckets == fc.num_buckets, 'all features must share the same buckets, but are %d and %s' % ( - num_buckets, str(fc)) + num_buckets, + str(fc), + ) fea_dict['sparse_fea'] = (tmp_vals % num_buckets, tmp_lens) if len(self._dense_fea_names) > 0: @@ -283,7 +301,8 @@ def _build(self, mode, params): self._dense_fea_cfgs, self._reserve_fields, drop_remainder, - need_pack=self._need_pack) + need_pack=self._need_pack, + ) for input_file in my_files: self._file_que.put(input_file) @@ -342,7 +361,7 @@ def _get_for_predictor(self, fea_dict): out_dict = { 'feature': { 'ragged_ids': fea_dict['feature']['sparse_fea'][0], - 'ragged_lens': fea_dict['feature']['sparse_fea'][1] + 'ragged_lens': fea_dict['feature']['sparse_fea'][1], } } if self._is_predictor and self._reserve_fields is not None: @@ -366,8 +385,11 @@ def _input_fn(mode=None, params=None, config=None): else, return: tf.estimator.export.ServingInputReceiver instance """ - if mode in (tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL, - tf.estimator.ModeKeys.PREDICT): + if mode in ( + tf.estimator.ModeKeys.TRAIN, + tf.estimator.ModeKeys.EVAL, + tf.estimator.ModeKeys.PREDICT, + ): # build dataset from self._config.input_path self._mode = mode dataset = self._build(mode, params) @@ -381,11 +403,14 @@ def _input_fn(mode=None, params=None, config=None): tf.int32, [None], name='ragged_lens') inputs = {'ragged_ids': ragged_ids, 'ragged_lens': ragged_lens} if self._has_ev: - features = {'ragged_ids': ragged_ids, 'ragged_lens': ragged_lens} + features = { + 'ragged_ids': ragged_ids, + 'ragged_lens': ragged_lens, + } else: features = { 'ragged_ids': ragged_ids % self._feature_configs[0].num_buckets, - 'ragged_lens': ragged_lens + 'ragged_lens': ragged_lens, } if len(self._dense_fea_names) > 0: inputs['dense_fea'] = array_ops.placeholder( diff --git a/easy_rec/python/input/parquet_input_v2.py b/easy_rec/python/input/parquet_input_v2.py index dba54498c..c9d963b11 100644 --- a/easy_rec/python/input/parquet_input_v2.py +++ b/easy_rec/python/input/parquet_input_v2.py @@ -21,18 +21,27 @@ class ParquetInputV2(ParquetInput): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - **kwargs): - super(ParquetInputV2, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config, **kwargs) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + **kwargs, + ): + super(ParquetInputV2, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + **kwargs, + ) self._need_pack = False def _predictor_preprocess(self, input_dict): @@ -116,7 +125,10 @@ def _preprocess(self, inputs=None): else: if fc.raw_input_dim > 1: input_vals = array_ops.placeholder( - dtypes.float32, [None, fc.raw_input_dim], name=input_name0) + dtypes.float32, + [None, fc.raw_input_dim], + name=input_name0, + ) else: input_vals = array_ops.placeholder( dtypes.float32, [None], name=input_name0) @@ -163,8 +175,11 @@ def _input_fn(mode=None, params=None, config=None): else, return: tf.estimator.export.ServingInputReceiver instance """ - if mode in (tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL, - tf.estimator.ModeKeys.PREDICT): + if mode in ( + tf.estimator.ModeKeys.TRAIN, + tf.estimator.ModeKeys.EVAL, + tf.estimator.ModeKeys.PREDICT, + ): # build dataset from self._config.input_path self._mode = mode dataset = self._build(mode, params) diff --git a/easy_rec/python/input/parquet_input_v3.py b/easy_rec/python/input/parquet_input_v3.py index 300a5bd1e..9431daed0 100644 --- a/easy_rec/python/input/parquet_input_v3.py +++ b/easy_rec/python/input/parquet_input_v3.py @@ -8,15 +8,15 @@ from easy_rec.python.utils.input_utils import get_type_defaults try: - from tensorflow.python.data.experimental.ops import parquet_dataset_ops - from tensorflow.python.data.experimental.ops import parquet_pybind - from tensorflow.python.data.experimental.ops import dataframe + from tensorflow.python.data.experimental.ops import ( # NOQA + dataframe, parquet_dataset_ops, parquet_pybind, + ) from tensorflow.python.ops import gen_ragged_conversion_ops from tensorflow.python.ops.work_queue import WorkQueue + _has_deep_rec = True except Exception: _has_deep_rec = False - pass if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -24,20 +24,28 @@ class ParquetInputV3(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None, - **kwargs): + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + **kwargs, + ): if not _has_deep_rec: raise RuntimeError('You should install DeepRec first.') - super(ParquetInputV3, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + super(ParquetInputV3, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) self._ignore_val_dict = {} for f in data_config.input_fields: @@ -74,13 +82,16 @@ def _ignore_and_cast(self, name, value): indices = tf.where(tf.equal(value.values, ignore_value)) value = tf.SparseTensor( tf.gather_nd(value.indices, indices), - tf.gather_nd(value.values, indices), value.dense_shape) + tf.gather_nd(value.values, indices), + value.dense_shape, + ) elif isinstance(value, tf.Tensor): indices = tf.where(tf.not_equal(value, ignore_value), name='indices') value = tf.SparseTensor( indices=indices, values=tf.gather_nd(value, indices), - dense_shape=tf.shape(value, out_type=tf.int64)) + dense_shape=tf.shape(value, out_type=tf.int64), + ) dtype = self._true_type_dict.get(name, None) if dtype: value = tf.cast(value, dtype) @@ -92,9 +103,11 @@ def _parse_dataframe_value(self, value): value.values.set_shape([None]) sparse_value = gen_ragged_conversion_ops.ragged_tensor_to_sparse( value.nested_row_splits, value.values) - return tf.SparseTensor(sparse_value.sparse_indices, - sparse_value.sparse_values, - sparse_value.sparse_dense_shape) + return tf.SparseTensor( + sparse_value.sparse_indices, + sparse_value.sparse_values, + sparse_value.sparse_dense_shape, + ) def _parse_dataframe(self, df): inputs = {} @@ -127,12 +140,12 @@ def _build(self, mode, params): task_index = max(self._task_index - 1, 0) task_num = max(self._task_num - 1, 1) - if self._data_config.pai_worker_queue and \ - mode == tf.estimator.ModeKeys.TRAIN: + if self._data_config.pai_worker_queue and mode == tf.estimator.ModeKeys.TRAIN: work_queue = WorkQueue( input_files, num_epochs=self.num_epochs, - shuffle=self._data_config.shuffle) + shuffle=self._data_config.shuffle, + ) my_files = work_queue.input_dataset() else: my_files = [] @@ -164,7 +177,8 @@ def _build(self, mode, params): batch_size=self._batch_size, fields=selected_fields, drop_remainder=self._data_config.drop_remainder, - num_parallel_reads=num_parallel_reads) + num_parallel_reads=num_parallel_reads, + ) # partition_count=task_num, # partition_index=task_index) @@ -173,20 +187,23 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True) + reshuffle_each_iteration=True, + ) dataset = dataset.repeat(self.num_epochs) else: dataset = dataset.repeat(1) dataset = dataset.map( self._parse_dataframe, - num_parallel_calls=self._data_config.num_parallel_calls) + num_parallel_calls=self._data_config.num_parallel_calls, + ) # preprocess is necessary to transform data # so that they could be feed into FeatureColumns dataset = dataset.map( map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls) + num_parallel_calls=self._data_config.num_parallel_calls, + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) diff --git a/easy_rec/python/input/rtp_input.py b/easy_rec/python/input/rtp_input.py index 9f9679b9e..1d8feb016 100644 --- a/easy_rec/python/input/rtp_input.py +++ b/easy_rec/python/input/rtp_input.py @@ -6,11 +6,13 @@ from easy_rec.python.input.input import Input from easy_rec.python.ops.gen_str_avx_op import str_split_by_chr -from easy_rec.python.utils.check_utils import check_split -from easy_rec.python.utils.check_utils import check_string_to_number from easy_rec.python.utils.input_utils import string_to_number from easy_rec.python.utils.tf_utils import get_tf_type +from easy_rec.python.utils.check_utils import ( # NOQA + check_split, check_string_to_number, +) + if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -30,17 +32,25 @@ class RTPInput(Input): columns are labels """ - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(RTPInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(RTPInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) logging.info('input_fields: %s label_fields: %s' % (','.join(self._input_fields), ','.join(self._label_fields))) self._rtp_separator = self._data_config.rtp_separator @@ -58,18 +68,21 @@ def _parse_csv(self, line): # the actual features are in one single column record_defaults[self._feature_col_id] = self._data_config.separator.join([ - str(self.get_type_defaults(t, v)) - for x, t, v in zip(self._input_fields, self._input_field_types, - self._input_field_defaults) - if x not in self._label_fields + str(self.get_type_defaults(t, v)) for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) if x not in self._label_fields ]) - check_list = [ + check_list = ([ tf.py_func( - check_split, [line, self._rtp_separator, - len(record_defaults)], - Tout=tf.bool) - ] if self._check_mode else [] + check_split, + [line, self._rtp_separator, + len(record_defaults)], + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): fields = tf.string_split(line, self._rtp_separator, skip_empty=False) @@ -97,13 +110,14 @@ def _parse_csv(self, line): # assume that the last field is the generated feature column print('field_delim = %s' % self._data_config.separator) feature_str = fields[:, self._feature_col_id] - check_list = [ + check_list = ([ tf.py_func( check_split, [feature_str, self._data_config.separator, len(record_types)], - Tout=tf.bool) - ] if self._check_mode else [] + Tout=tf.bool, + ) + ] if self._check_mode else []) with tf.control_dependencies(check_list): fields = str_split_by_chr( feature_str, self._data_config.separator, skip_empty=False) @@ -142,9 +156,10 @@ def _build(self, mode, params): for line_str in fin: line_tok = line_str.strip().split(self._rtp_separator) if self._num_cols != -1: - assert self._num_cols == len(line_tok), \ - 'num selected cols is %d, not equal to %d, current line is: %s, please check rtp_separator and data.' % \ - (self._num_cols, len(line_tok), line_str) + assert self._num_cols == len(line_tok), ( + 'num selected cols is %d, not equal to %d, current line is: %s,' + ' please check rtp_separator and data.') % ( + self._num_cols, len(line_tok), line_str) self._num_cols = len(line_tok) num_lines += 1 if num_lines > 10: @@ -161,10 +176,11 @@ def _build(self, mode, params): # the features are in one single column record_defaults.append( self._data_config.separator.join([ - str(self.get_type_defaults(t, v)) - for x, t, v in zip(self._input_fields, self._input_field_types, - self._input_field_defaults) - if x not in self._label_fields + str(self.get_type_defaults(t, v)) for x, t, v in zip( + self._input_fields, + self._input_field_types, + self._input_field_defaults, + ) if x not in self._label_fields ])) num_parallel_calls = self._data_config.num_parallel_calls @@ -186,7 +202,8 @@ def _build(self, mode, params): dataset = dataset.interleave( tf.data.TextLineDataset, cycle_length=parallel_num, - num_parallel_calls=parallel_num) + num_parallel_calls=parallel_num, + ) if not self._data_config.file_shard: dataset = self._safe_shard(dataset) @@ -195,7 +212,8 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True) + reshuffle_each_iteration=True, + ) dataset = dataset.repeat(self.num_epochs) else: logging.info('eval files[%d]: %s' % @@ -213,7 +231,8 @@ def _build(self, mode, params): # so that they could be feed into FeatureColumns dataset = dataset.map( map_func=self._preprocess, - num_parallel_calls=self._data_config.num_parallel_calls) + num_parallel_calls=self._data_config.num_parallel_calls, + ) dataset = dataset.prefetch(buffer_size=self._prefetch_size) diff --git a/easy_rec/python/input/rtp_input_v2.py b/easy_rec/python/input/rtp_input_v2.py index 32e841f8d..0987810b6 100644 --- a/easy_rec/python/input/rtp_input_v2.py +++ b/easy_rec/python/input/rtp_input_v2.py @@ -17,17 +17,25 @@ class RTPInputV2(Input): the original rtp format, it is not efficient for training, the performance have to be tuned. """ - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(RTPInputV2, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(RTPInputV2, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) def _parse_rtp(self, lines): tf_types = [tf.string for x in self._input_field_types] @@ -51,8 +59,10 @@ def _parse_one_line_tf(line): def_val = self.get_type_defaults(self._input_field_types[i], self._input_field_defaults[i]) temp_vals[i] = tf.cond( - tf.reduce_any(msk), lambda: tf.reduce_join(val, separator=','), - lambda: tf.constant(str(def_val))) + tf.reduce_any(msk), + lambda: tf.reduce_join(val, separator=','), + lambda: tf.constant(str(def_val)), + ) return temp_vals fields = tf.map_fn( @@ -60,7 +70,8 @@ def _parse_one_line_tf(line): lines, tf_types, parallel_iterations=64, - name='parse_one_line_tf_map_fn') + name='parse_one_line_tf_map_fn', + ) def _convert(x, target_type, name): if target_type in [DatasetConfig.FLOAT, DatasetConfig.DOUBLE]: @@ -111,7 +122,8 @@ def _build(self, mode, params): dataset = dataset.interleave( tf.data.TextLineDataset, cycle_length=parallel_num, - num_parallel_calls=parallel_num) + num_parallel_calls=parallel_num, + ) if not self._data_config.file_shard: dataset = self._safe_shard(dataset) @@ -120,7 +132,8 @@ def _build(self, mode, params): dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True) + reshuffle_each_iteration=True, + ) dataset = dataset.repeat(self.num_epochs) else: logging.info('eval files[%d]: %s' % diff --git a/easy_rec/python/input/tfrecord_input.py b/easy_rec/python/input/tfrecord_input.py index b9e25eef0..ae7b9cf72 100644 --- a/easy_rec/python/input/tfrecord_input.py +++ b/easy_rec/python/input/tfrecord_input.py @@ -13,17 +13,25 @@ class TFRecordInput(Input): - def __init__(self, - data_config, - feature_config, - input_path, - task_index=0, - task_num=1, - check_mode=False, - pipeline_config=None): - super(TFRecordInput, - self).__init__(data_config, feature_config, input_path, task_index, - task_num, check_mode, pipeline_config) + def __init__( + self, + data_config, + feature_config, + input_path, + task_index=0, + task_num=1, + check_mode=False, + pipeline_config=None, + ): + super(TFRecordInput, self).__init__( + data_config, + feature_config, + input_path, + task_index, + task_num, + check_mode, + pipeline_config, + ) self.feature_desc = {} for x, t, d in zip(self._input_fields, self._input_field_types, @@ -64,13 +72,15 @@ def _build(self, mode, params): lambda x: tf.data.TFRecordDataset( x, compression_type=data_compression_type), cycle_length=parallel_num, - num_parallel_calls=parallel_num) + num_parallel_calls=parallel_num, + ) dataset = dataset.shard(self._task_num, self._task_index) if self._data_config.shuffle: dataset = dataset.shuffle( self._data_config.shuffle_buffer_size, seed=2020, - reshuffle_each_iteration=True) + reshuffle_each_iteration=True, + ) dataset = dataset.repeat(self.num_epochs) else: logging.info('eval files[%d]: %s' % diff --git a/easy_rec/python/layers/backbone.py b/easy_rec/python/layers/backbone.py index e77ea1da5..2e644f26c 100644 --- a/easy_rec/python/layers/backbone.py +++ b/easy_rec/python/layers/backbone.py @@ -21,6 +21,7 @@ class Package(object): """A sub DAG of tf ops for reuse.""" + __packages = {} @staticmethod @@ -91,14 +92,21 @@ def __init__(self, config, features, input_layer, l2_reg=None): input_fn = self._input_layer.get_raw_features(self._features, group) input_feature_groups[group] = input_fn else: # embedding_layer - inputs, vocab, weights = self._input_layer.get_bucketized_features( + ( + inputs, + vocab, + weights, + ) = self._input_layer.get_bucketized_features( self._features, group) block.embedding_layer.vocab_size = vocab params = Parameter.make_from_pb(block.embedding_layer) input_fn = EmbeddingLayer(params, block.name) input_feature_groups[group] = (inputs, vocab, weights) - logging.info('add an embedding layer %s with vocab size %d', - block.name, vocab) + logging.info( + 'add an embedding layer %s with vocab size %d', + block.name, + vocab, + ) self._name_to_layer[block.name] = input_fn else: self.define_layers(layer, block, block.name, reuse) @@ -364,11 +372,13 @@ def load_keras_layer(self, layer_conf, name, reuse=None): has_reuse = True try: from funcsigs import signature + sig = signature(layer_cls.__init__) has_reuse = 'reuse' in sig.parameters.keys() except ImportError: try: from sklearn.externals.funcsigs import signature + sig = signature(layer_cls.__init__) has_reuse = 'reuse' in sig.parameters.keys() except ImportError: @@ -536,6 +546,7 @@ def merge_inputs(inputs, axis=-1, msg=''): return inputs[0] from functools import reduce + if all(map(lambda x: type(x) == list, inputs)): # merge multiple lists into a list return reduce(lambda x, y: x + y, inputs) diff --git a/easy_rec/python/layers/capsule_layer.py b/easy_rec/python/layers/capsule_layer.py index 22c8363cd..8519ae07a 100644 --- a/easy_rec/python/layers/capsule_layer.py +++ b/easy_rec/python/layers/capsule_layer.py @@ -35,8 +35,9 @@ def squash(self, inputs): """Squash inputs over the last dimension.""" input_norm = tf.reduce_sum(tf.square(inputs), keep_dims=True, axis=-1) input_norm_eps = tf.maximum(input_norm, 1e-8) - scale_factor = tf.pow(input_norm_eps / (1 + input_norm_eps), self._squash_pow) * \ - self._scale_ratio / tf.sqrt(input_norm_eps) + scale_factor = ( + tf.pow(input_norm_eps / (1 + input_norm_eps), self._squash_pow) * + self._scale_ratio / tf.sqrt(input_norm_eps)) tf.summary.histogram('capsule/squash_scale_factor', scale_factor) return scale_factor * inputs @@ -53,8 +54,8 @@ def _build_capsule_simi(self, high_capsules, capsule_num): simi = tf.reduce_sum(simi, axis=1) / div is_multi = tf.to_float(capsule_num > 1) - avg_simi = tf.reduce_sum((simi + 1) * is_multi) / \ - (2.0 * tf.reduce_sum(is_multi)) + avg_simi = tf.reduce_sum( + (simi + 1) * is_multi) / (2.0 * tf.reduce_sum(is_multi)) return avg_simi def __call__(self, seq_feas, seq_lens): @@ -70,11 +71,17 @@ def __call__(self, seq_feas, seq_lens): # pad or clip to max_seq_len seq_feas = tf.cond( tf.greater(tf.shape(seq_feas)[1], self._max_seq_len), - lambda: seq_feas[:, :self._max_seq_len, :], lambda: tf.cond( - tf.less(tf.shape(seq_feas)[1], self._max_seq_len), lambda: tf.pad( - seq_feas, [[0, 0], [ - 0, self._max_seq_len - tf.shape(seq_feas)[1] - ], [0, 0]]), lambda: seq_feas)) + lambda: seq_feas[:, :self._max_seq_len, :], + lambda: tf.cond( + tf.less(tf.shape(seq_feas)[1], self._max_seq_len), + lambda: tf.pad( + seq_feas, + [[0, 0], [0, self._max_seq_len - tf.shape(seq_feas)[1]], [0, 0] + ], + ), + lambda: seq_feas, + ), + ) seq_lens = tf.minimum(seq_lens, self._max_seq_len) batch_size = tf.shape(seq_lens)[0] @@ -82,14 +89,17 @@ def __call__(self, seq_feas, seq_lens): if self._is_training: routing_logits = tf.truncated_normal( [batch_size, self._max_seq_len, self._max_k], - stddev=self._routing_logits_stddev) + stddev=self._routing_logits_stddev, + ) else: np.random.seed(28) routing_logits = tf.constant( np.random.uniform( high=self._routing_logits_stddev, - size=[self._max_seq_len, self._max_k]), - dtype=tf.float32) + size=[self._max_seq_len, self._max_k], + ), + dtype=tf.float32, + ) routing_logits = tf.tile(routing_logits[None, :, :], [batch_size, 1, 1]) routing_logits = tf.stop_gradient(routing_logits) # batch_size x max_seq_len x max_k(bsh) @@ -139,17 +149,24 @@ def __call__(self, seq_feas, seq_lens): # batch_size x max_k x high_dim(bse,bsh->bhe) high_capsules = tf.einsum( - 'bse, bsh->bhe', seq_feas_high_stop - if iter_id + 1 < self._num_iters else seq_feas_high, routing_logits) + 'bse, bsh->bhe', + seq_feas_high_stop + if iter_id + 1 < self._num_iters else seq_feas_high, + routing_logits, + ) if iter_id + 1 == self._num_iters: capsule_simi = self._build_capsule_simi(high_capsules, num_high_capsules) tf.summary.scalar('caspule/simi_%d' % iter_id, capsule_simi) - tf.summary.scalar('capsule/before_squash', - tf.reduce_mean(tf.norm(high_capsules, axis=-1))) + tf.summary.scalar( + 'capsule/before_squash', + tf.reduce_mean(tf.norm(high_capsules, axis=-1)), + ) high_capsules = self.squash(high_capsules) - tf.summary.scalar('capsule/after_squash', - tf.reduce_mean(tf.norm(high_capsules, axis=-1))) + tf.summary.scalar( + 'capsule/after_squash', + tf.reduce_mean(tf.norm(high_capsules, axis=-1)), + ) capsule_simi_final = self._build_capsule_simi(high_capsules, num_high_capsules) tf.summary.scalar('caspule/simi_final', capsule_simi_final) diff --git a/easy_rec/python/layers/cmbf.py b/easy_rec/python/layers/cmbf.py index e5f1caeb2..3faf1dca2 100644 --- a/easy_rec/python/layers/cmbf.py +++ b/easy_rec/python/layers/cmbf.py @@ -50,18 +50,21 @@ def __init__(self, model_config, feature_configs, features, cmbf_config, if fea_group.group_name == 'general': self._general_feature_num = len(fea_group.feature_names) general_feature_names = set(fea_group.feature_names) - assert self._general_feature_num == len(general_feature_names), ( - 'there are duplicate features in `general` feature group') + assert self._general_feature_num == len( + general_feature_names + ), 'there are duplicate features in `general` feature group' elif fea_group.group_name == 'image': self._img_feature_num = len(fea_group.feature_names) img_feature_names = set(fea_group.feature_names) - assert self._img_feature_num == len(img_feature_names), ( - 'there are duplicate features in `image` feature group') + assert self._img_feature_num == len( + img_feature_names + ), 'there are duplicate features in `image` feature group' elif fea_group.group_name == 'text': txt_seq_feature_names = set(fea_group.feature_names) self._txt_feature_num = len(fea_group.feature_names) - assert self._txt_feature_num == len(txt_seq_feature_names), ( - 'there are duplicate features in `text` feature group') + assert self._txt_feature_num == len( + txt_seq_feature_names + ), 'there are duplicate features in `text` feature group' max_seq_len = 0 txt_fea_emb_dim_list = [] @@ -85,29 +88,29 @@ def __init__(self, model_config, feature_configs, features, cmbf_config, max_seq_len = feature_config.max_seq_len unique_dim_num = len(set(txt_fea_emb_dim_list)) - assert unique_dim_num <= 1 and len( - txt_fea_emb_dim_list - ) == self._txt_feature_num, ( - 'CMBF requires that all `text` feature dimensions must be consistent.') + assert ( + unique_dim_num <= 1 and + len(txt_fea_emb_dim_list) == self._txt_feature_num + ), 'CMBF requires that all `text` feature dimensions must be consistent.' unique_dim_num = len(set(general_emb_dim_list)) - assert unique_dim_num <= 1 and len( - general_emb_dim_list - ) == self._general_feature_num, ( - 'CMBF requires that all `general` feature dimensions must be consistent.' - ) + assert ( + unique_dim_num <= 1 and + len(general_emb_dim_list) == self._general_feature_num + ), 'CMBF requires that all `general` feature dimensions must be consistent.' unique_dim_num = len(set(img_fea_emb_dim_list)) - assert unique_dim_num <= 1 and len( - img_fea_emb_dim_list - ) == self._img_feature_num, ( - 'CMBF requires that all `image` feature dimensions must be consistent.') + assert ( + unique_dim_num <= 1 and + len(img_fea_emb_dim_list) == self._img_feature_num + ), 'CMBF requires that all `image` feature dimensions must be consistent.' if cmbf_config.use_position_embeddings: assert cmbf_config.max_position_embeddings > 0, ( 'model config `max_position_embeddings` must be greater than 0. ' 'It must be set when `use_position_embeddings` is true (default)') assert cmbf_config.max_position_embeddings >= max_seq_len, ( - 'model config `max_position_embeddings` must be greater than or equal to the maximum of all feature config ' - '`max_seq_len`, which is %d' % max_seq_len) + 'model config `max_position_embeddings` must be greater than' + ' or equal to the maximum of all feature config ' + '`max_seq_len`, which is %d') % max_seq_len self._img_emb_size = img_fea_emb_dim_list[0] if img_fea_emb_dim_list else 0 self._txt_emb_size = txt_fea_emb_dim_list[0] if txt_fea_emb_dim_list else 0 @@ -123,8 +126,11 @@ def __init__(self, model_config, feature_configs, features, cmbf_config, self._txt_self_attention_layer_num = cmbf_config.text_self_attention_layer_num self._cross_modal_layer_num = cmbf_config.cross_modal_layer_num print('txt_feature_num: {0}, img_feature_num: {1}, txt_seq_feature_num: {2}' - .format(self._general_feature_num, self._img_feature_num, - len(self._txt_seq_features) if self._txt_seq_features else 0)) + .format( + self._general_feature_num, + self._img_feature_num, + len(self._txt_seq_features) if self._txt_seq_features else 0, + )) print('txt_embedding_size: {0}, img_embedding_size: {1}'.format( self._txt_emb_size, self._img_emb_size)) if self._img_features is not None: @@ -170,9 +176,9 @@ def image_self_attention_tower(self): if self._img_patch_num > 1: # image feature dimension: patch_num * emb_size img_fea_num = self._img_patch_num img_emb_size = self._img_emb_size // self._img_patch_num - assert img_emb_size * self._img_patch_num == self._img_emb_size, ( - 'image feature dimension must equal to `image_feature_slice_num * embedding_size_per_region`' - ) + assert ( + img_emb_size * self._img_patch_num == self._img_emb_size + ), 'image feature dimension must equal to `image_feature_slice_num * embedding_size_per_region`' self._img_emb_size = img_emb_size if self._img_emb_size != hidden_size: # Run a linear projection of `hidden_size` @@ -201,7 +207,7 @@ def image_self_attention_tower(self): hidden_dropout_prob=self._model_config.hidden_dropout_prob, attention_probs_dropout_prob=self._model_config .attention_probs_dropout_prob, - name='image_self_attention' + name='image_self_attention', ) # shape: [batch_size, image_seq_num/image_feature_dim, hidden_size] # print('img_attention_fea:', img_attention_fea.shape) return img_attention_fea @@ -258,12 +264,14 @@ def dynamic_mask(x, max_len): use_position_embeddings=self._model_config.use_position_embeddings, max_position_embeddings=self._model_config.max_position_embeddings, position_embedding_name='position_embeddings_%d' % i, - dropout_prob=self._model_config.text_seq_emb_dropout_prob) + dropout_prob=self._model_config.text_seq_emb_dropout_prob, + ) all_txt_features.append(seq_fea) input_mask = tf.map_fn( fn=lambda t: dynamic_mask(t, max_seq_len), - elems=tf.to_int32(seq_len)) + elems=tf.to_int32(seq_len), + ) input_masks.append(input_mask) txt_features = tf.concat(all_txt_features, axis=1) @@ -284,7 +292,7 @@ def dynamic_mask(x, max_len): hidden_dropout_prob=self._model_config.hidden_dropout_prob, attention_probs_dropout_prob=self._model_config .attention_probs_dropout_prob, - name='text_self_attention' + name='text_self_attention', ) # shape: [batch_size, txt_seq_length, hidden_size] print('txt_attention_fea:', txt_attention_fea.shape) return txt_attention_fea, input_mask, input_masks @@ -296,8 +304,11 @@ def merge_text_embedding(self, txt_embeddings, input_masks): text_seq_emb = [] if self._general_feature_num > 0: - text_emb = tf.slice(txt_embeddings, [0, 0, 0], - [shape[0], self._general_feature_num, shape[2]]) + text_emb = tf.slice( + txt_embeddings, + [0, 0, 0], + [shape[0], self._general_feature_num, shape[2]], + ) text_seq_emb.append(text_emb) begin = self._general_feature_num @@ -329,6 +340,7 @@ def __call__(self, is_training, *args, **kwargs): if not is_training: self._model_config.hidden_dropout_prob = 0.0 self._model_config.attention_probs_dropout_prob = 0.0 + self._model_config.text_seq_emb_dropout_prob = 0.0 # shape: [batch_size, image_num/image_dim, hidden_size] img_attention_fea = self.image_self_attention_tower() @@ -339,7 +351,10 @@ def __call__(self, is_training, *args, **kwargs): all_fea = [] if None not in [img_attention_fea, txt_attention_fea]: - img_embeddings, txt_embeddings = multihead_cross_attention.cross_attention_tower( + ( + img_embeddings, + txt_embeddings, + ) = multihead_cross_attention.cross_attention_tower( img_attention_fea, txt_attention_fea, num_hidden_layers=self._cross_modal_layer_num, @@ -353,10 +368,11 @@ def __call__(self, is_training, *args, **kwargs): self._head_num, hidden_dropout_prob=self._model_config.hidden_dropout_prob, attention_probs_dropout_prob=self._model_config - .attention_probs_dropout_prob) - # img_embeddings shape: [batch_size, image_(region_)num/image_feature_dim, multi_head_num * image_cross_head_size] + .attention_probs_dropout_prob, + ) + # img_emb shape: [batch_size, image_(region_)num/image_feature_dim, multi_head_num * image_cross_head_size] print('img_embeddings:', img_embeddings.shape) - # txt_embeddings shape: [batch_size, general_feature_num + max_txt_seq_len, multi_head_num * text_cross_head_size] + # txt_emb shape: [batch_size, general_feature_num + max_txt_seq_len, multi_head_num * text_cross_head_size] print('txt_embeddings:', txt_embeddings.shape) # shape: [batch_size, multi_head_num * image_cross_head_size] @@ -379,8 +395,12 @@ def __call__(self, is_training, *args, **kwargs): if self._other_features is not None: if self._model_config.HasField('other_feature_dnn'): l2_reg = kwargs['l2_reg'] if 'l2_reg' in kwargs else 0 - other_dnn_layer = dnn.DNN(self._model_config.other_feature_dnn, l2_reg, - 'other_dnn', is_training) + other_dnn_layer = dnn.DNN( + self._model_config.other_feature_dnn, + l2_reg, + 'other_dnn', + is_training, + ) other_fea = other_dnn_layer(self._other_features) all_fea.append(other_fea) # e.g. statistical features else: diff --git a/easy_rec/python/layers/common_layers.py b/easy_rec/python/layers/common_layers.py index 68ecf37f5..dfe57ddfc 100644 --- a/easy_rec/python/layers/common_layers.py +++ b/easy_rec/python/layers/common_layers.py @@ -11,14 +11,16 @@ tf = tf.compat.v1 -def highway(x, - size=None, - activation=None, - num_layers=1, - scope='highway', - dropout=0.0, - init_gate_bias=-1.0, - reuse=None): +def highway( + x, + size=None, + activation=None, + num_layers=1, + scope='highway', + dropout=0.0, + init_gate_bias=-1.0, + reuse=None, +): if isinstance(activation, six.string_types): activation = get_activation(activation) with tf.variable_scope(scope, reuse): @@ -35,7 +37,8 @@ def highway(x, activation=tf.sigmoid, bias_initializer=initializer, name='gate_%d' % i, - reuse=reuse) + reuse=reuse, + ) H = tf.layers.dense( x, size, activation=activation, name='activation_%d' % i, reuse=reuse) if dropout > 0.0: @@ -44,11 +47,13 @@ def highway(x, return x -def text_cnn(x, - filter_sizes=(3, 4, 5), - num_filters=(128, 64, 64), - scope_name='textcnn', - reuse=False): +def text_cnn( + x, + filter_sizes=(3, 4, 5), + num_filters=(128, 64, 64), + scope_name='textcnn', + reuse=False, +): # x: None * step_dim * embed_dim assert len(filter_sizes) == len(num_filters) initializer = tf.variance_scaling_initializer() @@ -67,7 +72,8 @@ def text_cnn(x, name='conv_layer', reuse=reuse, kernel_initializer=initializer, - padding='same') + padding='same', + ) pool = tf.reduce_max( conv, axis=1) # max pooling, shape: (batch_size, num_filters) pooled_outputs.append(pool) @@ -83,7 +89,8 @@ def layer_norm(input_tensor, name=None, reuse=None): begin_norm_axis=-1, begin_params_axis=-1, reuse=reuse, - scope=name) + scope=name, + ) class EnhancedInputLayer(object): diff --git a/easy_rec/python/layers/dnn.py b/easy_rec/python/layers/dnn.py index 7a57f5661..9587fb358 100644 --- a/easy_rec/python/layers/dnn.py +++ b/easy_rec/python/layers/dnn.py @@ -12,13 +12,15 @@ class DNN: - def __init__(self, - dnn_config, - l2_reg, - name='dnn', - is_training=False, - last_layer_no_activation=False, - last_layer_no_batch_norm=False): + def __init__( + self, + dnn_config, + l2_reg, + name='dnn', + is_training=False, + last_layer_no_activation=False, + last_layer_no_batch_norm=False, + ): """Initializes a `DNN` Layer. Args: @@ -59,14 +61,16 @@ def __call__(self, deep_fea, hidden_layer_feature_output=False): units=unit, kernel_regularizer=self._l2_reg, activation=None, - name='%s/dnn_%d' % (self._name, i)) + name='%s/dnn_%d' % (self._name, i), + ) if self._config.use_bn and ((i + 1 < hidden_units_len) or not self._last_layer_no_batch_norm): deep_fea = tf.layers.batch_normalization( deep_fea, training=self._is_training, trainable=True, - name='%s/dnn_%d/bn' % (self._name, i)) + name='%s/dnn_%d/bn' % (self._name, i), + ) if (i + 1 < hidden_units_len) or not self._last_layer_no_activation: deep_fea = self.activation( deep_fea, name='%s/dnn_%d/act' % (self._name, i)) @@ -76,11 +80,12 @@ def __call__(self, deep_fea, hidden_layer_feature_output=False): deep_fea = tf.nn.dropout( deep_fea, keep_prob=1 - self.dropout_ratio[i], - name='%s/%d/dropout' % (self._name, i)) + name='%s/%d/dropout' % (self._name, i), + ) if hidden_layer_feature_output: hidden_feature_dict['hidden_layer' + str(i)] = deep_fea - if (i + 1 == hidden_units_len): + if i + 1 == hidden_units_len: hidden_feature_dict['hidden_layer_end'] = deep_fea return hidden_feature_dict else: diff --git a/easy_rec/python/layers/embed_input_layer.py b/easy_rec/python/layers/embed_input_layer.py index 284593fa0..63250de4b 100644 --- a/easy_rec/python/layers/embed_input_layer.py +++ b/easy_rec/python/layers/embed_input_layer.py @@ -15,8 +15,8 @@ def __init__(self, feature_groups_config, dump_dir=None): self._dump_dir = dump_dir def __call__(self, features, group_name): - assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % \ - ','.join([x for x in self._feature_groups]) + assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ','.join( + [x for x in self._feature_groups]) feature_group = self._feature_groups[group_name] group_features = [] for feature_name in feature_group.feature_names: diff --git a/easy_rec/python/layers/input_layer.py b/easy_rec/python/layers/input_layer.py index 27bc9bdf4..19f6e4202 100644 --- a/easy_rec/python/layers/input_layer.py +++ b/easy_rec/python/layers/input_layer.py @@ -13,15 +13,17 @@ from easy_rec.python.compat.feature_column import feature_column from easy_rec.python.feature_column.feature_column import FeatureColumnParser from easy_rec.python.feature_column.feature_group import FeatureGroup -from easy_rec.python.layers import sequence_feature_layer -from easy_rec.python.layers import variational_dropout_layer from easy_rec.python.layers.keras import TextCNN from easy_rec.python.layers.utils import Parameter from easy_rec.python.protos.feature_config_pb2 import WideOrDeep from easy_rec.python.utils import conditional from easy_rec.python.utils import shape_utils -from easy_rec.python.compat.feature_column.feature_column_v2 import is_embedding_column # NOQA +from easy_rec.python.compat.feature_column.feature_column_v2 import ( # NOQA + is_embedding_column,) +from easy_rec.python.layers import ( # NOQA + sequence_feature_layer, variational_dropout_layer, +) class InputLayer(object): @@ -30,22 +32,30 @@ class InputLayer(object): This class apply feature_columns to input tensors to generate wide features and deep features. """ - def __init__(self, - feature_configs, - feature_groups_config, - variational_dropout_config=None, - wide_output_dim=-1, - ev_params=None, - embedding_regularizer=None, - kernel_regularizer=None, - is_training=False, - is_predicting=False): + def __init__( + self, + feature_configs, + feature_groups_config, + variational_dropout_config=None, + wide_output_dim=-1, + ev_params=None, + embedding_regularizer=None, + kernel_regularizer=None, + is_training=False, + is_predicting=False, + ): self._feature_groups = { x.group_name: FeatureGroup(x) for x in feature_groups_config } self.sequence_feature_layer = sequence_feature_layer.SequenceFeatureLayer( - feature_configs, feature_groups_config, ev_params, - embedding_regularizer, kernel_regularizer, is_training, is_predicting) + feature_configs, + feature_groups_config, + ev_params, + embedding_regularizer, + kernel_regularizer, + is_training, + is_predicting, + ) self._seq_feature_groups_config = [] for x in feature_groups_config: for y in x.sequence_features: @@ -102,7 +112,8 @@ def get_combined_feature(self, features, group_name, is_dict=False): group_seq_arr, feature_name_to_output_tensors, negative_sampler=negative_sampler, - scope_name=group_name) + scope_name=group_name, + ) group_features.extend(all_seq_fea) for col, fea in zip(group_seq_arr, all_seq_fea): feature_name_to_output_tensors['seq_fea/' + col.group_name] = fea @@ -126,7 +137,9 @@ def get_plain_feature(self, features, group_name): group_features: list of features """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups])) + group_name, + ','.join([x for x in self._feature_groups]), + ) feature_group = self._feature_groups[group_name] group_columns, _ = feature_group.select_columns(self._fc_parser) @@ -138,7 +151,8 @@ def get_plain_feature(self, features, group_name): features, group_columns, cols_to_output_tensors=cols_to_output_tensors, - is_training=self._is_training) + is_training=self._is_training, + ) group_features = [cols_to_output_tensors[x] for x in group_columns] embedding_reg_lst = [] @@ -164,7 +178,9 @@ def get_sequence_feature(self, features, group_name): 1d sequence length tensor. """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups])) + group_name, + ','.join([x for x in self._feature_groups]), + ) if self._variational_dropout_config is not None: raise ValueError( @@ -202,7 +218,9 @@ def get_raw_features(self, features, group_name): features: all raw features in list """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups])) + group_name, + ','.join([x for x in self._feature_groups]), + ) feature_group = self._feature_groups[group_name] return [features[x] for x in feature_group.feature_names] @@ -217,7 +235,9 @@ def get_bucketized_features(self, features, group_name): features: all raw features in list, added feature offset """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups])) + group_name, + ','.join([x for x in self._feature_groups]), + ) feature_group = self._feature_groups[group_name] offset = 0 values = [] @@ -263,7 +283,9 @@ def __call__(self, features, group_name, is_combine=True, is_dict=False): 1 dimension sequence length tensor. """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups])) + group_name, + ','.join([x for x in self._feature_groups]), + ) if is_combine: return self.get_combined_feature(features, group_name, is_dict) @@ -294,7 +316,9 @@ def single_call_input_layer(self, group_features: list of features """ assert group_name in self._feature_groups, 'invalid group_name[%s], list: %s' % ( - group_name, ','.join([x for x in self._feature_groups])) + group_name, + ','.join([x for x in self._feature_groups]), + ) feature_group = self._feature_groups[group_name] group_columns, group_seq_columns = feature_group.select_columns( self._fc_parser) @@ -304,7 +328,8 @@ def single_call_input_layer(self, group_columns, cols_to_output_tensors=cols_to_output_tensors, feature_name_to_output_tensors=feature_name_to_output_tensors, - is_training=self._is_training) + is_training=self._is_training, + ) embedding_reg_lst = [] builder = feature_column._LazyBuilder(features) @@ -327,9 +352,10 @@ def single_call_input_layer(self, kernel_regularizer=self._kernel_regularizer, use_bias=False, activation=None, - name='attention') + name='attention', + ) attn_logits = tf.squeeze(attn_logits, axis=-1) - attn_logits_padding = tf.ones_like(attn_logits) * (-2**32 + 1) + attn_logits_padding = tf.ones_like(attn_logits) * (-(2**32) + 1) seq_mask = tf.sequence_mask(seq_len) attn_score = tf.nn.softmax( tf.where(seq_mask, attn_logits, attn_logits_padding)) @@ -356,15 +382,17 @@ def single_call_input_layer(self, self._variational_dropout_config, features_dimension, self._is_training, - name=group_name) + name=group_name, + ) concat_features = variational_dropout(concat_features) group_features = tf.split( concat_features, list(features_dimension.values()), axis=-1) else: concat_features = array_ops.concat( [output_features] + seq_features, axis=-1) - group_features = [cols_to_output_tensors[x] for x in group_columns] + \ - [cols_to_output_tensors[x] for x in group_seq_columns] + group_features = [cols_to_output_tensors[x] for x in group_columns] + [ + cols_to_output_tensors[x] for x in group_seq_columns + ] if self._embedding_regularizer is not None: for fc, val in cols_to_output_tensors.items(): diff --git a/easy_rec/python/layers/keras/__init__.py b/easy_rec/python/layers/keras/__init__.py index 17e7cdb1c..7afbc8ebb 100644 --- a/easy_rec/python/layers/keras/__init__.py +++ b/easy_rec/python/layers/keras/__init__.py @@ -5,11 +5,6 @@ from .blocks import Highway from .blocks import TextCNN from .bst import BST -from .custom_ops import EditDistance -from .custom_ops import MappedDotProduct -from .custom_ops import OverlapFeature -from .custom_ops import SeqAugmentOps -from .custom_ops import TextNormalize from .data_augment import SeqAugment from .din import DIN from .embedding import EmbeddingLayer @@ -25,10 +20,15 @@ from .multi_head_attention import MultiHeadAttention from .multi_task import AITMTower from .multi_task import MMoE -from .numerical_embedding import AutoDisEmbedding -from .numerical_embedding import NaryDisEmbedding -from .numerical_embedding import PeriodicEmbedding from .ppnet import PPNet from .transformer import TextEncoder from .transformer import TransformerBlock from .transformer import TransformerEncoder + +from .custom_ops import ( # NOQA + EditDistance, MappedDotProduct, OverlapFeature, SeqAugmentOps, + TextNormalize, +) +from .numerical_embedding import ( # NOQA + AutoDisEmbedding, NaryDisEmbedding, PeriodicEmbedding, +) diff --git a/easy_rec/python/layers/keras/activation.py b/easy_rec/python/layers/keras/activation.py index fa6218e64..d83351638 100644 --- a/easy_rec/python/layers/keras/activation.py +++ b/easy_rec/python/layers/keras/activation.py @@ -56,7 +56,8 @@ def build(self, input_shape): shape=(input_shape[-1],), initializer=Zeros(), dtype=tf.float32, - name='dice_alpha') # name='alpha_'+self.name + name='dice_alpha', + ) # name='alpha_'+self.name super(Dice, self).build(input_shape) # Be sure to call this somewhere! self.uses_learning_phase = True diff --git a/easy_rec/python/layers/keras/attention.py b/easy_rec/python/layers/keras/attention.py index 4831ccae8..08e867eb7 100644 --- a/easy_rec/python/layers/keras/attention.py +++ b/easy_rec/python/layers/keras/attention.py @@ -5,6 +5,7 @@ This file follows the terminology of https://arxiv.org/abs/1706.03762 Figure 2. Attention is formed by three tensors: Query, Key and Value. """ + import tensorflow as tf from tensorflow.python.keras.layers import Layer diff --git a/easy_rec/python/layers/keras/blocks.py b/easy_rec/python/layers/keras/blocks.py index c9e722a67..23ca4f445 100644 --- a/easy_rec/python/layers/keras/blocks.py +++ b/easy_rec/python/layers/keras/blocks.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Convenience blocks for building models.""" + import logging import tensorflow as tf @@ -47,8 +48,18 @@ def __init__(self, params, name='mlp', reuse=None, **kwargs): logging.info( 'MLP(%s) units: %s, dropout: %r, activate=%s, use_bn=%r, final_bn=%r,' ' final_activate=%s, bias=%r, initializer=%s, bn_after_activation=%r' % - (name, units, dropout_rate, activation, use_bn, use_final_bn, - final_activation, use_bias, initializer, use_bn_after_act)) + ( + name, + units, + dropout_rate, + activation, + use_bn, + use_final_bn, + final_activation, + use_bias, + initializer, + use_bn_after_act, + )) assert len(units) > 0, 'MLP(%s) takes at least one hidden units' % name self.reuse = reuse self.add_to_outputs = params.get_or_default('add_to_outputs', False) @@ -58,27 +69,45 @@ def __init__(self, params, name='mlp', reuse=None, **kwargs): for i, num_units in enumerate(units[:-1]): name = 'layer_%d' % i drop_rate = dropout_rate[i] if i < num_dropout else 0.0 - self.add_rich_layer(num_units, use_bn, drop_rate, activation, initializer, - use_bias, use_bn_after_act, name, - params.l2_regularizer) + self.add_rich_layer( + num_units, + use_bn, + drop_rate, + activation, + initializer, + use_bias, + use_bn_after_act, + name, + params.l2_regularizer, + ) n = len(units) - 1 drop_rate = dropout_rate[n] if num_dropout > n else 0.0 name = 'layer_%d' % n - self.add_rich_layer(units[-1], use_final_bn, drop_rate, final_activation, - initializer, use_final_bias, use_bn_after_act, name, - params.l2_regularizer) + self.add_rich_layer( + units[-1], + use_final_bn, + drop_rate, + final_activation, + initializer, + use_final_bias, + use_bn_after_act, + name, + params.l2_regularizer, + ) - def add_rich_layer(self, - num_units, - use_bn, - dropout_rate, - activation, - initializer, - use_bias, - use_bn_after_activation, - name, - l2_reg=None): + def add_rich_layer( + self, + num_units, + use_bn, + dropout_rate, + activation, + initializer, + use_bias, + use_bn_after_activation, + name, + l2_reg=None, + ): act_layer = activation_layer(activation, name='%s/act' % name) if use_bn and not use_bn_after_activation: dense = Dense( @@ -86,7 +115,8 @@ def add_rich_layer(self, use_bias=use_bias, kernel_initializer=initializer, kernel_regularizer=l2_reg, - name='%s/dense' % name) + name='%s/dense' % name, + ) self._sub_layers.append(dense) bn = tf.keras.layers.BatchNormalization( name='%s/bn' % name, trainable=True) @@ -98,7 +128,8 @@ def add_rich_layer(self, use_bias=use_bias, kernel_initializer=initializer, kernel_regularizer=l2_reg, - name='%s/dense' % name) + name='%s/dense' % name, + ) self._sub_layers.append(dense) self._sub_layers.append(act_layer) if use_bn and use_bn_after_activation: @@ -158,7 +189,8 @@ def build(self, input_shape): units=dim, bias_initializer=self.gate_bias_initializer, activation='sigmoid', - name='gate_%d' % i) + name='gate_%d' % i, + ) self.gates.append(gate) self.transforms.append(Dense(units=dim)) @@ -231,7 +263,8 @@ def __init__(self, params, name='text_cnn', reuse=None, **kwargs): conv = tf.keras.layers.Conv1D( filters=int(filters), kernel_size=int(size), - activation=self.config.activation) + activation=self.config.activation, + ) self.conv_layers.append(conv) if self.config.HasField('mlp'): p = Parameter.make_from_pb(self.config.mlp) diff --git a/easy_rec/python/layers/keras/bst.py b/easy_rec/python/layers/keras/bst.py index dbd4882ed..ee4dc0561 100644 --- a/easy_rec/python/layers/keras/bst.py +++ b/easy_rec/python/layers/keras/bst.py @@ -19,15 +19,14 @@ def __init__(self, params, name='bst', reuse=None, **kwargs): self.l2_reg = params.l2_regularizer self.config = params.get_pb_config() - def encode(self, seq_input, max_position): + def encode(self, seq_input, seq_mask, max_position, hidden_dropout, + attention_dropout, target_pos): seq_fea = multihead_cross_attention.embedding_postprocessor( seq_input, position_embedding_name=self.name, max_position_embeddings=max_position, - reuse_position_embedding=self.reuse) - - n = tf.count_nonzero(seq_input, axis=-1) - seq_mask = tf.cast(n > 0, tf.int32) + reuse_position_embedding=self.reuse, + dropout_prob=hidden_dropout) attention_mask = multihead_cross_attention.create_attention_mask_from_input_mask( from_tensor=seq_fea, to_mask=seq_mask) @@ -41,24 +40,29 @@ def encode(self, seq_input, max_position): attention_mask=attention_mask, intermediate_size=self.config.intermediate_size, intermediate_act_fn=hidden_act, - hidden_dropout_prob=self.config.hidden_dropout_prob, - attention_probs_dropout_prob=self.config.attention_probs_dropout_prob, + hidden_dropout_prob=hidden_dropout, + attention_probs_dropout_prob=attention_dropout, initializer_range=self.config.initializer_range, name=self.name + '/transformer', - reuse=self.reuse) + reuse=self.reuse, + ) # attention_fea shape: [batch_size, seq_length, hidden_size] if self.config.output_all_token_embeddings: out_fea = tf.reshape(attention_fea, [-1, max_position * self.config.hidden_size]) - else: + elif target_pos == 'head': out_fea = attention_fea[:, 0, :] # target feature + else: + out_fea = attention_fea[:, -1, :] # target feature print('bst output shape:', out_fea.shape) return out_fea def call(self, inputs, training=None, **kwargs): - if not training: - self.config.hidden_dropout_prob = 0.0 - self.config.attention_probs_dropout_prob = 0.0 + hidden_dropout = self.config.hidden_dropout_prob if training else 0.0 + attention_dropout = self.config.attention_probs_dropout_prob if training else 0.0 + tf.logging.info( + 'BST config hidden_dropout_prob=%f, attention_probs_dropout_prob=%f', + hidden_dropout, attention_dropout) assert isinstance(inputs, (list, tuple)) assert len(inputs) >= 2 # seq_input: [batch_size, seq_len, embed_size] @@ -73,14 +77,45 @@ def call(self, inputs, training=None, **kwargs): max_position, message='sequence length is greater than `max_position_embeddings`:' + str(max_position) + ' in feature group:' + self.name + - ', you should set `max_seq_len` in sequence feature configs') + ', you should set `max_seq_len` in sequence feature configs', + ) + orig_mask = tf.sequence_mask( + seq_len, maxlen=cur_batch_max_seq_len, dtype=tf.int32) + keep_target = self.config.target_item_position in ('head', 'tail') if self.config.output_all_token_embeddings: seq_input = tf.cond( - tf.constant(max_position) > cur_batch_max_seq_len, lambda: tf.pad( - seq_input, [[0, 0], [0, max_position - cur_batch_max_seq_len], - [0, 0]], 'CONSTANT'), - lambda: tf.slice(seq_input, [0, 0, 0], [-1, max_position, -1])) + tf.constant(max_position) > cur_batch_max_seq_len, + lambda: tf.pad( + seq_input, + [[0, 0], [0, max_position - cur_batch_max_seq_len], [0, 0]], + 'CONSTANT', + ), + lambda: tf.slice(seq_input, [0, 0, 0], [-1, max_position, -1]), + ) + orig_mask = tf.cond( + tf.constant(max_position) > cur_batch_max_seq_len, + lambda: tf.pad(orig_mask, [[0, 0], + [0, max_position - cur_batch_max_seq_len]], + 'CONSTANT'), + lambda: tf.slice(orig_mask, [0, 0], [-1, max_position]), + ) + # 再根据 target 拼接方式，对 orig_mask 做相同的 concat/pad + if target is not None and keep_target: + # target 位置一般是有效的 1，或者你也可以选择 0 + target_mask = tf.ones([batch_size, 1], dtype=tf.int32) + if self.config.target_item_position == 'head': + orig_mask = tf.concat([target_mask, orig_mask], axis=1) + else: + orig_mask = tf.concat([orig_mask, target_mask], axis=1) + elif self.config.reserve_target_position: + target_mask = tf.ones([batch_size, 1], dtype=tf.int32) + if self.config.target_item_position == 'head': + orig_mask = tf.concat([target_mask, orig_mask], axis=1) + else: + orig_mask = tf.concat([orig_mask, target_mask], axis=1) + + seq_mask = orig_mask if seq_embed_size != self.config.hidden_size: seq_input = tf.layers.dense( @@ -89,13 +124,14 @@ def call(self, inputs, training=None, **kwargs): activation=tf.nn.relu, kernel_regularizer=self.l2_reg, name=self.name + '/seq_project', - reuse=self.reuse) + reuse=self.reuse, + ) - keep_target = self.config.target_item_position in ('head', 'tail') if target is not None and keep_target: target_size = target.shape.as_list()[-1] - assert seq_embed_size == target_size, 'the embedding size of sequence and target item is not equal' \ - ' in feature group:' + self.name + assert seq_embed_size == target_size, ( + 'the embedding size of sequence and target item is not equal' + ' in feature group:' + self.name) if target_size != self.config.hidden_size: target = tf.layers.dense( target, @@ -103,7 +139,8 @@ def call(self, inputs, training=None, **kwargs): activation=tf.nn.relu, kernel_regularizer=self.l2_reg, name=self.name + '/target_project', - reuse=self.reuse) + reuse=self.reuse, + ) # target_feature: [batch_size, 1, embed_size] target = tf.expand_dims(target, 1) # seq_input: [batch_size, seq_len+1, embed_size] @@ -116,4 +153,5 @@ def call(self, inputs, training=None, **kwargs): max_position += 1 with tf.control_dependencies([valid_len]): - return self.encode(seq_input, max_position) + return self.encode(seq_input, seq_mask, max_position, hidden_dropout, + attention_dropout, self.config.target_item_position) diff --git a/easy_rec/python/layers/keras/custom_ops.py b/easy_rec/python/layers/keras/custom_ops.py index c215ee332..2b8cf0c7d 100644 --- a/easy_rec/python/layers/keras/custom_ops.py +++ b/easy_rec/python/layers/keras/custom_ops.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Convenience blocks for using custom ops.""" + import logging import os @@ -59,10 +60,14 @@ def call(self, inputs, training=None, **kwargs): 'mask', (embedding_dim,), dtype=tf.float32, trainable=True) seq_len = tf.to_int32(seq_len) with ops.device('/CPU:0'): - aug_seq, aug_len = self.seq_augment(seq_input, seq_len, mask_emb, - self.seq_aug_params.crop_rate, - self.seq_aug_params.reorder_rate, - self.seq_aug_params.mask_rate) + aug_seq, aug_len = self.seq_augment( + seq_input, + seq_len, + mask_emb, + self.seq_aug_params.crop_rate, + self.seq_aug_params.reorder_rate, + self.seq_aug_params.mask_rate, + ) return aug_seq, aug_len @@ -107,7 +112,8 @@ def __init__(self, params, name='mapped_dot_product', reuse=None, **kwargs): self.embedding_table = tf.get_variable( name='dot_product_emb_table', shape=[vocab_size, self.emb_dim], - dtype=tf.float32) + dtype=tf.float32, + ) def call(self, inputs, training=None, **kwargs): query, doc = inputs[:2] @@ -117,26 +123,31 @@ def call(self, inputs, training=None, **kwargs): document=doc, feature_name=self.name, separator=self.separator, - default_value=self.default_value) + default_value=self.default_value, + ) tf.summary.scalar(self.name, tf.reduce_mean(feature)) if self.print_first_n: encode_q = tf.regex_replace(query, self.separator, ' ') encode_t = tf.regex_replace(query, self.separator, ' ') feature = tf.Print( - feature, [encode_q, encode_t, feature], + feature, + [encode_q, encode_t, feature], message=self.name, first_n=self.print_first_n, - summarize=self.summarize) + summarize=self.summarize, + ) if self.norm_fn is not None: fn = eval(self.norm_fn) feature = fn(feature) tf.summary.scalar('normalized_%s' % self.name, tf.reduce_mean(feature)) if self.print_first_n: feature = tf.Print( - feature, [feature], + feature, + [feature], message='normalized %s' % self.name, first_n=self.print_first_n, - summarize=self.summarize) + summarize=self.summarize, + ) if self.boundaries: feature = self.bucketize(feature, boundaries=self.boundaries) tf.summary.histogram('bucketized_%s' % self.name, feature) @@ -169,7 +180,8 @@ def __init__(self, params, name='overlap_feature', reuse=None, **kwargs): self.embedding_table = tf.get_variable( name='overlap_emb_table', shape=[vocab_size, self.emb_dim], - dtype=tf.float32) + dtype=tf.float32, + ) def call(self, inputs, training=None, **kwargs): query, title = inputs[:2] @@ -182,7 +194,8 @@ def call(self, inputs, training=None, **kwargs): default_value=self.default_value, boundaries=self.boundaries, methods=self.methods, - dtype=tf.int32 if self.boundaries else tf.float32) + dtype=tf.int32 if self.boundaries else tf.float32, + ) for i, method in enumerate(self.methods): # warning: feature[:, i] may be not the result of method @@ -194,10 +207,12 @@ def call(self, inputs, training=None, **kwargs): encode_q = tf.regex_replace(query, self.separator, ' ') encode_t = tf.regex_replace(query, self.separator, ' ') feature = tf.Print( - feature, [encode_q, encode_t, feature], + feature, + [encode_q, encode_t, feature], message=self.name, first_n=self.print_first_n, - summarize=self.summarize) + summarize=self.summarize, + ) if self.norm_fn is not None: fn = eval(self.norm_fn) feature = fn(feature) @@ -244,7 +259,8 @@ def call(self, inputs, training=None, **kwargs): input2, normalize=False, dtype=tf.int32, - encoding=self.txt_encoding) + encoding=self.txt_encoding, + ) ids = tf.clip_by_value(dist, 0, self.emb_size - 1) embed = tf.nn.embedding_lookup(self.embedding_table, ids) return embed diff --git a/easy_rec/python/layers/keras/data_augment.py b/easy_rec/python/layers/keras/data_augment.py index a11f08120..93674d130 100644 --- a/easy_rec/python/layers/keras/data_augment.py +++ b/easy_rec/python/layers/keras/data_augment.py @@ -39,8 +39,10 @@ def item_crop(aug_data, length, crop_rate): y = zeros[:max_length - num_left] cropped = tf.concat([x, y], axis=0) cropped_item_seq = tf.where( - crop_begin + num_left < max_length, cropped, - tf.concat([aug_data[crop_begin:], zeros[:crop_begin]], axis=0)) + crop_begin + num_left < max_length, + cropped, + tf.concat([aug_data[crop_begin:], zeros[:crop_begin]], axis=0), + ) return cropped_item_seq, num_left @@ -95,8 +97,10 @@ def reorder_fn(): return tf.cond(tf.equal(method, 0), trans_fn[0], trans_fn[1]) aug_seq, aug_len = tf.cond( - tf.equal(method, 0), crop_fn, - lambda: tf.cond(tf.equal(method, 1), mask_fn, reorder_fn)) + tf.equal(method, 0), + crop_fn, + lambda: tf.cond(tf.equal(method, 1), mask_fn, reorder_fn), + ) return aug_seq, aug_len @@ -105,7 +109,8 @@ def sequence_augment(seq_input, seq_len, mask, aug_param): aug_seq, aug_len = tf.map_fn( lambda elems: augment_fn(elems, aug_param, mask), elems=(seq_input, lengths), - dtype=(tf.float32, tf.int32)) + dtype=(tf.float32, tf.int32), + ) aug_seq = tf.reshape(aug_seq, tf.shape(seq_input)) return aug_seq, aug_len diff --git a/easy_rec/python/layers/keras/din.py b/easy_rec/python/layers/keras/din.py index 082677e0b..1f671b80f 100644 --- a/easy_rec/python/layers/keras/din.py +++ b/easy_rec/python/layers/keras/din.py @@ -32,7 +32,11 @@ def call(self, inputs, training=None, **kwargs): if query_emb_size != seq_emb_size: logging.info( ' the embedding size of sequence [%d] and target item [%d] is not equal' - ' in feature group: %s', seq_emb_size, query_emb_size, self.name) + ' in feature group: %s', + seq_emb_size, + query_emb_size, + self.name, + ) if query_emb_size < seq_emb_size: query = tf.pad(query, [[0, 0], [0, seq_emb_size - query_emb_size]]) else: @@ -47,7 +51,7 @@ def call(self, inputs, training=None, **kwargs): seq_mask = tf.sequence_mask(seq_len, max_seq_len, dtype=tf.bool) seq_mask = tf.expand_dims(seq_mask, 1) - paddings = tf.ones_like(scores) * (-2**32 + 1) + paddings = tf.ones_like(scores) * (-(2**32) + 1) scores = tf.where(seq_mask, scores, paddings) # [B, 1, L] if self.config.attention_normalizer == 'softmax': scores = tf.nn.softmax(scores) # (B, 1, L) diff --git a/easy_rec/python/layers/keras/einsum_dense.py b/easy_rec/python/layers/keras/einsum_dense.py index 7531644dc..eab9aabf3 100644 --- a/easy_rec/python/layers/keras/einsum_dense.py +++ b/easy_rec/python/layers/keras/einsum_dense.py @@ -4,12 +4,12 @@ import string import tensorflow as tf -from tensorflow.python.keras import activations -from tensorflow.python.keras import constraints -from tensorflow.python.keras import initializers -from tensorflow.python.keras import regularizers from tensorflow.python.keras.layers import Layer +from tensorflow.python.keras import ( # NOQA + activations, constraints, initializers, regularizers, +) + class EinsumDense(Layer): """A layer that uses `einsum` as the backing computation. @@ -105,19 +105,21 @@ class EinsumDense(Layer): (None, 32, 64) """ - def __init__(self, - equation, - output_shape, - activation=None, - bias_axes=None, - kernel_initializer='glorot_uniform', - bias_initializer='zeros', - kernel_regularizer=None, - bias_regularizer=None, - kernel_constraint=None, - bias_constraint=None, - lora_rank=None, - **kwargs): + def __init__( + self, + equation, + output_shape, + activation=None, + bias_axes=None, + kernel_initializer='glorot_uniform', + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + lora_rank=None, + **kwargs, + ): super(EinsumDense, self).__init__(**kwargs) self.equation = equation if isinstance(output_shape, int): @@ -327,7 +329,8 @@ def _check_load_own_variables(self, store): name=self.name, num_var=len(store.keys()), num_key=len(store.keys()), - names=[v.name for v in all_vars])) + names=[v.name for v in all_vars], + )) def _get_kernel_with_merged_lora(self): kernel_value = self.kernel @@ -409,15 +412,15 @@ def _analyze_split_string(split_string, input_shape_at_dim = input_shape[input_dim_map[dim]] if dim in output_dim_map: output_shape_at_dim = output_shape[output_dim_map[dim]] - if (output_shape_at_dim is not None and - output_shape_at_dim != input_shape_at_dim): + if output_shape_at_dim is not None and output_shape_at_dim != input_shape_at_dim: raise ValueError( 'Input shape and output shape do not match at shared ' "dimension '{dim}'. Input shape is {input_shape_at_dim}, " 'and output shape is {output_shape}.'.format( dim=dim, input_shape_at_dim=input_shape_at_dim, - output_shape=output_shape[output_dim_map[dim]])) + output_shape=output_shape[output_dim_map[dim]], + )) for dim in output_spec: if dim not in input_spec and dim not in weight_spec: diff --git a/easy_rec/python/layers/keras/embedding.py b/easy_rec/python/layers/keras/embedding.py index 77b513951..5ce61fa4a 100644 --- a/easy_rec/python/layers/keras/embedding.py +++ b/easy_rec/python/layers/keras/embedding.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Fused embedding layer.""" + import tensorflow as tf from tensorflow.python.keras.layers import Embedding from tensorflow.python.keras.layers import Layer @@ -72,9 +73,13 @@ def call(self, inputs, training=None, **kwargs): batch_size = tf.shape(inputs[i])[0] embeddings[i] = tf.cond( tf.equal(tf.size(embeddings[i]), 0), - lambda: tf.zeros([batch_size, self.embed_dim]), lambda: _combine( + lambda: tf.zeros([batch_size, self.embed_dim]), + lambda: _combine( tf.reshape(embeddings[i], [batch_size, -1, self.embed_dim]), - weights[i], self.combine_fn)) + weights[i], + self.combine_fn, + ), + ) if self.do_concat: embeddings = tf.concat(embeddings, axis=-1) print('Embedding layer:', self.name, embeddings) diff --git a/easy_rec/python/layers/keras/fibinet.py b/easy_rec/python/layers/keras/fibinet.py index 220c57cb5..23c4b6729 100644 --- a/easy_rec/python/layers/keras/fibinet.py +++ b/easy_rec/python/layers/keras/fibinet.py @@ -55,7 +55,8 @@ def build(self, input_shape): units=reduction_size, activation='relu', kernel_initializer='he_normal', - name='W1') + name='W1', + ) self.excite_layer = Dense( units=emb_size, kernel_initializer='glorot_normal', name='W2') super(SENet, self).build(input_shape) # Be sure to call this somewhere! diff --git a/easy_rec/python/layers/keras/interaction.py b/easy_rec/python/layers/keras/interaction.py index 9b14f254a..6f177e0db 100644 --- a/easy_rec/python/layers/keras/interaction.py +++ b/easy_rec/python/layers/keras/interaction.py @@ -120,7 +120,8 @@ def call(self, inputs, **kwargs): activations = tf.where( condition=tf.cast(upper_tri_mask, tf.bool), x=tf.zeros_like(xactions), - y=xactions) + y=xactions, + ) out_dim = num_features * num_features else: activations = tf.boolean_mask(xactions, lower_tri_mask) @@ -326,9 +327,10 @@ def __init__(self, params, name='cin', reuse=None, **kwargs): self._hidden_feature_sizes = list( params.get_or_default('hidden_feature_sizes', [])) - assert isinstance(self._hidden_feature_sizes, list) and len( - self._hidden_feature_sizes - ) > 0, 'parameter hidden_feature_sizes must be a list of int with length greater than 0' + assert ( + isinstance(self._hidden_feature_sizes, list) and + len(self._hidden_feature_sizes) > 0 + ), 'parameter hidden_feature_sizes must be a list of int with length greater than 0' kernel_regularizer = params.get_or_default('kernel_regularizer', None) self._kernel_regularizer = tf.keras.regularizers.get(kernel_regularizer) @@ -349,12 +351,14 @@ def build(self, input_shape): tfv1.get_variable( name='cin_kernel_%d' % i, shape=[ - hidden_feature_sizes[i + 1], hidden_feature_sizes[i], - hidden_feature_sizes[0] + hidden_feature_sizes[i + 1], + hidden_feature_sizes[i], + hidden_feature_sizes[0], ], initializer=tf.initializers.he_normal(), regularizer=self._kernel_regularizer, - trainable=True) for i in range(len(self._hidden_feature_sizes)) + trainable=True, + ) for i in range(len(self._hidden_feature_sizes)) ] self.bias_list = [ tfv1.get_variable( @@ -362,7 +366,8 @@ def build(self, input_shape): shape=[hidden_feature_sizes[i + 1]], initializer=tf.keras.initializers.Zeros, regularizer=self._bias_regularizer, - trainable=True) for i in range(len(self._hidden_feature_sizes)) + trainable=True, + ) for i in range(len(self._hidden_feature_sizes)) ] super(CIN, self).build(input_shape) @@ -394,13 +399,15 @@ def call(self, input, **kwargs): feature_map_elementwise = tf.multiply( intermediate_tensor_expanded, - tf.expand_dims(tf.expand_dims(self.kernel_list[i], -1), 0)) + tf.expand_dims(tf.expand_dims(self.kernel_list[i], -1), 0), + ) feature_map = tf.reduce_sum( tf.reduce_sum(feature_map_elementwise, axis=3), axis=2) feature_map = tf.add( feature_map, - tf.expand_dims(tf.expand_dims(self.bias_list[i], axis=-1), axis=0)) + tf.expand_dims(tf.expand_dims(self.bias_list[i], axis=-1), axis=0), + ) feature_map = tf.nn.relu(feature_map) x_i = feature_map diff --git a/easy_rec/python/layers/keras/layer_norm.py b/easy_rec/python/layers/keras/layer_norm.py index 7d6c81d5f..10491140f 100644 --- a/easy_rec/python/layers/keras/layer_norm.py +++ b/easy_rec/python/layers/keras/layer_norm.py @@ -1,4 +1,5 @@ """Layer Normalization layer.""" + import tensorflow as tf from tensorflow.python.keras import constraints from tensorflow.python.keras import initializers @@ -168,18 +169,20 @@ class LayerNormalization(Layer): - [Lei Ba et al., 2016](https://arxiv.org/abs/1607.06450). """ - def __init__(self, - axis=-1, - epsilon=1e-3, - center=True, - scale=True, - beta_initializer='zeros', - gamma_initializer='ones', - beta_regularizer=None, - gamma_regularizer=None, - beta_constraint=None, - gamma_constraint=None, - **kwargs): + def __init__( + self, + axis=-1, + epsilon=1e-3, + center=True, + scale=True, + beta_initializer='zeros', + gamma_initializer='ones', + beta_regularizer=None, + gamma_regularizer=None, + beta_constraint=None, + gamma_constraint=None, + **kwargs, + ): super(LayerNormalization, self).__init__(**kwargs) if isinstance(axis, (list, tuple)): self.axis = list(axis) @@ -274,13 +277,13 @@ def call(self, inputs): broadcast_shape[dim] = input_shape.dims[dim].value def _broadcast(v): - if (v is not None and len(v.shape) != ndims and self.axis != [ndims - 1]): + if v is not None and len(v.shape) != ndims and self.axis != [ndims - 1]: return tf.reshape(v, broadcast_shape) return v if not self._fused: input_dtype = inputs.dtype - if (input_dtype in ('float16', 'bfloat16') and self.dtype == 'float32'): + if input_dtype in ('float16', 'bfloat16') and self.dtype == 'float32': # If mixed precision is used, cast inputs to float32 so that # this is at least as numerically stable as the fused version. inputs = tf.cast(inputs, 'float32') diff --git a/easy_rec/python/layers/keras/mask_net.py b/easy_rec/python/layers/keras/mask_net.py index bf687154e..96eb0fea5 100644 --- a/easy_rec/python/layers/keras/mask_net.py +++ b/easy_rec/python/layers/keras/mask_net.py @@ -54,7 +54,8 @@ def build(self, input_shape): activation='relu', kernel_initializer='he_uniform', kernel_regularizer=self.l2_reg, - name='aggregation') + name='aggregation', + ) self.weight_layer = Dense(input_dim, name='weights') if self._projection_dim is not None: logging.info('%s project dim is %d', self.name, self._projection_dim) @@ -62,7 +63,8 @@ def build(self, input_shape): self._projection_dim, kernel_regularizer=self.l2_reg, use_bias=False, - name='project') + name='project', + ) if self.config.input_layer_norm: # 推荐在调用MaskBlock之前做好 layer norm，否则每一次调用都需要对input做ln if tf.__version__ >= '2.0': diff --git a/easy_rec/python/layers/keras/multi_head_attention.py b/easy_rec/python/layers/keras/multi_head_attention.py index a5ca0b40d..5bbe0112a 100644 --- a/easy_rec/python/layers/keras/multi_head_attention.py +++ b/easy_rec/python/layers/keras/multi_head_attention.py @@ -209,7 +209,8 @@ def build(self, input_shape): [self._num_heads, self._key_dim]), bias_axes=bias_axes if self._use_bias else None, name='query', - **self._get_common_kwargs_for_sublayer()) + **self._get_common_kwargs_for_sublayer(), + ) self._query_dense.build(query_shape) einsum_equation, bias_axes, output_rank = _build_proj_equation( key_rank - 1, bound_dims=1, output_dims=2) @@ -219,7 +220,8 @@ def build(self, input_shape): [self._num_heads, self._key_dim]), bias_axes=bias_axes if self._use_bias else None, name='key', - **self._get_common_kwargs_for_sublayer()) + **self._get_common_kwargs_for_sublayer(), + ) self._key_dense.build(key_shape) einsum_equation, bias_axes, output_rank = _build_proj_equation( value_rank - 1, bound_dims=1, output_dims=2) @@ -229,7 +231,8 @@ def build(self, input_shape): [self._num_heads, self._value_dim]), bias_axes=bias_axes if self._use_bias else None, name='value', - **self._get_common_kwargs_for_sublayer()) + **self._get_common_kwargs_for_sublayer(), + ) self._value_dense.build(value_shape) # Builds the attention computations for multi-head dot product # attention. These computations could be wrapped into the keras @@ -309,7 +312,8 @@ def _make_output_dense(self, query_shape, common_kwargs, name=None): output_shape=_get_output_shape(output_rank - 1, output_shape), bias_axes=bias_axes if self._use_bias else None, name=name, - **common_kwargs) + **common_kwargs, + ) def _build_attention(self, rank): """Builds multi-head dot-product attention computations. @@ -505,9 +509,8 @@ def _compute_attention_mask( auto_mask = mask if auto_mask is None else auto_mask & mask if auto_mask is not None: # merge attention_mask & automatic mask, to shape [B, T, S] - attention_mask = ( - auto_mask if attention_mask is None else - tf.cast(attention_mask, tf.bool) & auto_mask) + attention_mask = auto_mask if attention_mask is None else tf.cast( + attention_mask, tf.bool) & auto_mask return attention_mask def _compute_causal_mask(self, query, value=None): @@ -559,7 +562,8 @@ def compute_output_shape(self, input_shape): query_shape=query_shape, value_shape=value_shape, query_last_dim=query_shape[-1], - value_last_dim=value_shape[-1])) + value_last_dim=value_shape[-1], + )) if value_shape[1:-1] != key_shape[1:-1]: raise ValueError( diff --git a/easy_rec/python/layers/keras/multi_task.py b/easy_rec/python/layers/keras/multi_task.py index dbb26ee86..6d3c97d22 100644 --- a/easy_rec/python/layers/keras/multi_task.py +++ b/easy_rec/python/layers/keras/multi_task.py @@ -41,7 +41,8 @@ def __init__(self, params, name='MMoE', reuse=None, **kwargs): self._num_expert, activation='softmax', name='gate_%d' % task_id, - kernel_regularizer=params.l2_regularizer) + kernel_regularizer=params.l2_regularizer, + ) self._gates.append(dense) def call(self, inputs, training=None, **kwargs): diff --git a/easy_rec/python/layers/keras/numerical_embedding.py b/easy_rec/python/layers/keras/numerical_embedding.py index 65cc77d52..dda436efe 100644 --- a/easy_rec/python/layers/keras/numerical_embedding.py +++ b/easy_rec/python/layers/keras/numerical_embedding.py @@ -161,13 +161,15 @@ def build(self, input_shape): 'coefficients', shape=[1, self.num_features, emb_dim], partitioner=partitioner, - initializer=self.initializer) + initializer=self.initializer, + ) if self.add_linear_layer: self.linear = NLinear( self.num_features, self.embedding_dim, self.embedding_dim, - name='nd_linear') + name='nd_linear', + ) super(PeriodicEmbedding, self).build(input_shape) def call(self, inputs, **kwargs): @@ -218,15 +220,18 @@ def build(self, input_shape): self.meta_emb = self.add_weight( 'meta_embedding', shape=[self.num_features, self.num_bins, self.emb_dim], - partitioner=partitioner) + partitioner=partitioner, + ) self.proj_w = self.add_weight( 'project_w', shape=[1, self.num_features, self.num_bins], - partitioner=partitioner) + partitioner=partitioner, + ) self.proj_mat = self.add_weight( 'project_mat', shape=[self.num_features, self.num_bins, self.num_bins], - partitioner=partitioner) + partitioner=partitioner, + ) super(AutoDisEmbedding, self).build(input_shape) def call(self, inputs, **kwargs): @@ -277,9 +282,15 @@ def __init__(self, params, name='nary_dis_embedding', reuse=None, **kwargs): self.output_tensor_list = params.get_or_default('output_tensor_list', False) logging.info( '{} carries: {}, lengths: {}, vocab_size: {}, intra_ary: {}, replicas: {}, multiplier: {}' - .format(self.name, ','.join(map(str, self.carries)), - ','.join(map(str, self.lengths)), self.vocab_size, - self.intra_ary_pooling, self.num_replicas, self.multiplier)) + .format( + self.name, + ','.join(map(str, self.carries)), + ','.join(map(str, self.lengths)), + self.vocab_size, + self.intra_ary_pooling, + self.num_replicas, + self.multiplier, + )) @staticmethod def max_length(carry): diff --git a/easy_rec/python/layers/keras/ppnet.py b/easy_rec/python/layers/keras/ppnet.py index 431034924..4f43198e5 100644 --- a/easy_rec/python/layers/keras/ppnet.py +++ b/easy_rec/python/layers/keras/ppnet.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Convenience blocks for building models.""" + import logging import tensorflow as tf @@ -53,7 +54,8 @@ def __init__(self, activation='sigmoid', use_bias=not do_batch_norm, kernel_initializer=initializer, - name='weight') + name='weight', + ) self._sub_layers.append(dense) self._sub_layers.append(lambda x: x * 2) @@ -102,8 +104,18 @@ def __init__(self, params, name='ppnet', reuse=None, **kwargs): logging.info( 'MLP(%s) units: %s, dropout: %r, activate=%s, use_bn=%r, final_bn=%r,' ' final_activate=%s, bias=%r, initializer=%s, bn_after_activation=%r' % - (name, units, dropout_rate, activation, use_bn, use_final_bn, - final_activation, use_bias, initializer, use_bn_after_act)) + ( + name, + units, + dropout_rate, + activation, + use_bn, + use_final_bn, + final_activation, + use_bias, + initializer, + use_bn_after_act, + )) assert len(units) > 0, 'MLP(%s) takes at least one hidden units' % name self.reuse = reuse @@ -115,32 +127,50 @@ def __init__(self, params, name='ppnet', reuse=None, **kwargs): for i, num_units in enumerate(units[:-1]): name = 'layer_%d' % i drop_rate = dropout_rate[i] if i < num_dropout else 0.0 - self.add_rich_layer(num_units, use_bn, drop_rate, activation, initializer, - use_bias, use_bn_after_act, name, - params.l2_regularizer) + self.add_rich_layer( + num_units, + use_bn, + drop_rate, + activation, + initializer, + use_bias, + use_bn_after_act, + name, + params.l2_regularizer, + ) self._sub_layers.append( GateNN(gate_params, num_units, 'gate_%d' % (i + 1))) n = len(units) - 1 drop_rate = dropout_rate[n] if num_dropout > n else 0.0 name = 'layer_%d' % n - self.add_rich_layer(units[-1], use_final_bn, drop_rate, final_activation, - initializer, use_final_bias, use_bn_after_act, name, - params.l2_regularizer) + self.add_rich_layer( + units[-1], + use_final_bn, + drop_rate, + final_activation, + initializer, + use_final_bias, + use_bn_after_act, + name, + params.l2_regularizer, + ) if mode == 'lazy': self._sub_layers.append( GateNN(gate_params, units[-1], 'gate_%d' % (n + 1))) - def add_rich_layer(self, - num_units, - use_bn, - dropout_rate, - activation, - initializer, - use_bias, - use_bn_after_activation, - name, - l2_reg=None): + def add_rich_layer( + self, + num_units, + use_bn, + dropout_rate, + activation, + initializer, + use_bias, + use_bn_after_activation, + name, + l2_reg=None, + ): act_layer = activation_layer(activation, name='%s/act' % name) if use_bn and not use_bn_after_activation: dense = tf.keras.layers.Dense( @@ -148,7 +178,8 @@ def add_rich_layer(self, use_bias=use_bias, kernel_initializer=initializer, kernel_regularizer=l2_reg, - name='%s/dense' % name) + name='%s/dense' % name, + ) self._sub_layers.append(dense) bn = tf.keras.layers.BatchNormalization( name='%s/bn' % name, trainable=True) @@ -160,7 +191,8 @@ def add_rich_layer(self, use_bias=use_bias, kernel_initializer=initializer, kernel_regularizer=l2_reg, - name='%s/dense' % name) + name='%s/dense' % name, + ) self._sub_layers.append(dense) self._sub_layers.append(act_layer) if use_bn and use_bn_after_activation: diff --git a/easy_rec/python/layers/keras/transformer.py b/easy_rec/python/layers/keras/transformer.py index d71a02831..23e8196f3 100644 --- a/easy_rec/python/layers/keras/transformer.py +++ b/easy_rec/python/layers/keras/transformer.py @@ -150,7 +150,8 @@ def __init__(self, params, name='text_encoder', reuse=None, **kwargs): self.vocab = tf.feature_column.categorical_column_with_vocabulary_file( 'tokens', vocabulary_file=vocab_file, - default_value=self.default_token_id) + default_value=self.default_token_id, + ) logging.info('vocab file of TextEncoder(%s) is %s', name, vocab_file) trans_params.vocab_size = self.vocab.vocabulary_size self.encoder = TransformerEncoder(trans_params, name='transformer') @@ -175,15 +176,19 @@ def call(self, inputs, training=None, **kwargs): token_ids, default_value=self.default_token_id, name='token_ids') length = tf.shape(token_ids)[-1] token_ids = tf.cond( - tf.less_equal(length, self.encoder.max_position), lambda: token_ids, - lambda: tf.slice(token_ids, [0, 0], [-1, self.encoder.max_position])) + tf.less_equal(length, self.encoder.max_position), + lambda: token_ids, + lambda: tf.slice(token_ids, [0, 0], [-1, self.encoder.max_position]), + ) mask = tf.not_equal(token_ids, self.default_token_id, name='mask') else: tokens = tf.sparse.to_dense(tokens, default_value='') length = tf.shape(tokens)[-1] tokens = tf.cond( - tf.less_equal(length, self.encoder.max_position), lambda: tokens, - lambda: tf.slice(tokens, [0, 0], [-1, self.encoder.max_position])) + tf.less_equal(length, self.encoder.max_position), + lambda: tokens, + lambda: tf.slice(tokens, [0, 0], [-1, self.encoder.max_position]), + ) token_ids = tf.string_to_hash_bucket_fast( tokens, self.encoder.vocab_size, name='token_ids') mask = tf.not_equal(tokens, '', name='mask') diff --git a/easy_rec/python/layers/layer_norm.py b/easy_rec/python/layers/layer_norm.py index 7e86963ad..d54bd75b9 100644 --- a/easy_rec/python/layers/layer_norm.py +++ b/easy_rec/python/layers/layer_norm.py @@ -17,13 +17,17 @@ def __init__(self, hidden_size, params={}): def build(self, _): self.scale = tf.get_variable( - 'layer_norm_scale', [self.hidden_size], + 'layer_norm_scale', + [self.hidden_size], initializer=tf.keras.initializers.Ones(), - dtype=tf.float32) + dtype=tf.float32, + ) self.bias = tf.get_variable( - 'layer_norm_bias', [self.hidden_size], + 'layer_norm_bias', + [self.hidden_size], initializer=tf.keras.initializers.Zeros(), - dtype=tf.float32) + dtype=tf.float32, + ) self.built = True def call(self, x): diff --git a/easy_rec/python/layers/mmoe.py b/easy_rec/python/layers/mmoe.py index 4e409a7c8..df56d6ed0 100644 --- a/easy_rec/python/layers/mmoe.py +++ b/easy_rec/python/layers/mmoe.py @@ -12,13 +12,15 @@ class MMOE: - def __init__(self, - expert_dnn_config, - l2_reg, - num_task, - num_expert=None, - name='mmoe', - is_training=False): + def __init__( + self, + expert_dnn_config, + l2_reg, + num_task, + num_expert=None, + name='mmoe', + is_training=False, + ): """Initializes a `DNN` Layer. Args: @@ -35,8 +37,9 @@ def __init__(self, self._expert_dnn_configs = expert_dnn_config self._num_expert = len(expert_dnn_config) else: - assert num_expert is not None and num_expert > 0, \ - 'param `num_expert` must be large than zero, when expert_dnn_config is not a list' + assert ( + num_expert is not None and num_expert > 0 + ), 'param `num_expert` must be large than zero, when expert_dnn_config is not a list' self._expert_dnn_configs = [expert_dnn_config] * num_expert self._num_expert = num_expert logging.info('num_expert: {0}'.format(self._num_expert)) @@ -55,7 +58,8 @@ def gate(self, unit, deep_fea, name): inputs=deep_fea, units=unit, kernel_regularizer=self._l2_reg, - name='%s/dnn' % name) + name='%s/dnn' % name, + ) fea = tf.nn.softmax(fea, axis=1) return fea @@ -67,7 +71,8 @@ def __call__(self, deep_fea): expert_dnn_config, self._l2_reg, name='%s/expert_%d' % (self._name, expert_id), - is_training=self._is_training) + is_training=self._is_training, + ) expert_fea = expert_dnn(deep_fea) expert_fea_list.append(expert_fea) experts_fea = tf.stack(expert_fea_list, axis=1) diff --git a/easy_rec/python/layers/multihead_attention.py b/easy_rec/python/layers/multihead_attention.py index 1da28e5d4..23443af5b 100644 --- a/easy_rec/python/layers/multihead_attention.py +++ b/easy_rec/python/layers/multihead_attention.py @@ -86,19 +86,22 @@ def _compute_qkv(self, q, k, v): self._head_num * self._head_size, use_bias=False, kernel_regularizer=self._l2_reg, - name='%s/%s/dnn' % (self._name, 'query')) + name='%s/%s/dnn' % (self._name, 'query'), + ) k = tf.layers.dense( k, self._head_num * self._head_size, use_bias=False, kernel_regularizer=self._l2_reg, - name='%s/%s/dnn' % (self._name, 'key')) + name='%s/%s/dnn' % (self._name, 'key'), + ) v = tf.layers.dense( v, self._head_num * self._head_size, use_bias=False, kernel_regularizer=self._l2_reg, - name='%s/%s/dnn' % (self._name, 'value')) + name='%s/%s/dnn' % (self._name, 'value'), + ) return q, k, v def _combine_heads(self, multi_head_tensor): @@ -124,8 +127,9 @@ def _multi_head_attention(self, attention_input): out: The output of multi head attention layer, has a shape of [bs, feature_num, head_num * head_size]. """ if isinstance(attention_input, list): - assert len(attention_input) == 3 or len(attention_input) == 1, \ - 'If the input of multi_head_attention is a list, the length must be 1 or 3.' + assert ( + len(attention_input) == 3 or len(attention_input) == 1 + ), 'If the input of multi_head_attention is a list, the length must be 1 or 3.' if len(attention_input) == 3: ori_q = attention_input[0] @@ -151,7 +155,8 @@ def _multi_head_attention(self, attention_input): out.shape[2], use_bias=False, kernel_regularizer=self._l2_reg, - name='%s/dnn' % (self._name)) + name='%s/dnn' % (self._name), + ) res_out = tf.nn.relu(out + W_0_x) return res_out else: diff --git a/easy_rec/python/layers/multihead_cross_attention.py b/easy_rec/python/layers/multihead_cross_attention.py index f230ac974..0f6b6ea69 100644 --- a/easy_rec/python/layers/multihead_cross_attention.py +++ b/easy_rec/python/layers/multihead_cross_attention.py @@ -39,21 +39,23 @@ def dropout(input_tensor, dropout_prob): return output -def attention_layer(from_tensor, - to_tensor, - size_per_head, - num_attention_heads=1, - attention_mask=None, - query_act=None, - key_act=None, - value_act=None, - attention_probs_dropout_prob=0.0, - initializer_range=0.02, - do_return_2d_tensor=False, - batch_size=None, - from_seq_length=None, - to_seq_length=None, - reuse=None): +def attention_layer( + from_tensor, + to_tensor, + size_per_head, + num_attention_heads=1, + attention_mask=None, + query_act=None, + key_act=None, + value_act=None, + attention_probs_dropout_prob=0.0, + initializer_range=0.02, + do_return_2d_tensor=False, + batch_size=None, + from_seq_length=None, + to_seq_length=None, + reuse=None, +): """Performs multi-headed attention from `from_tensor` to `to_tensor`. This is an implementation of multi-headed attention based on "Attention is all you Need". @@ -128,7 +130,7 @@ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, from_seq_length = from_shape[1] to_seq_length = to_shape[1] elif len(from_shape) == 2: - if (batch_size is None or from_seq_length is None or to_seq_length is None): + if batch_size is None or from_seq_length is None or to_seq_length is None: raise ValueError( 'When passing in rank 2 tensors to attention_layer, the values ' 'for `batch_size`, `from_seq_length`, and `to_seq_length` ' @@ -151,7 +153,8 @@ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, activation=query_act, name='query', kernel_initializer=create_initializer(initializer_range), - reuse=reuse) + reuse=reuse, + ) # `key_layer` = [B*T, N*H] key_layer = tf.layers.dense( @@ -160,7 +163,8 @@ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, activation=key_act, name='key', kernel_initializer=create_initializer(initializer_range), - reuse=reuse) + reuse=reuse, + ) # `value_layer` = [B*T, N*H] value_layer = tf.layers.dense( @@ -169,7 +173,8 @@ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, activation=value_act, name='value', kernel_initializer=create_initializer(initializer_range), - reuse=reuse) + reuse=reuse, + ) # `query_layer` = [B, N, F, H] query_layer = transpose_for_scores(query_layer, batch_size, @@ -226,28 +231,32 @@ def transpose_for_scores(input_tensor, batch_size, num_attention_heads, # `context_layer` = [B*F, N*H] context_layer = tf.reshape( context_layer, - [batch_size * from_seq_length, num_attention_heads * size_per_head]) + [batch_size * from_seq_length, num_attention_heads * size_per_head], + ) else: # `context_layer` = [B, F, N*H] context_layer = tf.reshape( context_layer, - [batch_size, from_seq_length, num_attention_heads * size_per_head]) + [batch_size, from_seq_length, num_attention_heads * size_per_head], + ) return context_layer -def transformer_encoder(input_tensor, - attention_mask=None, - hidden_size=768, - num_hidden_layers=12, - num_attention_heads=12, - intermediate_size=3072, - intermediate_act_fn=gelu, - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - initializer_range=0.02, - reuse=None, - name='transformer'): +def transformer_encoder( + input_tensor, + attention_mask=None, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + intermediate_act_fn=gelu, + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + initializer_range=0.02, + reuse=None, + name='transformer', +): """Multi-headed, multi-layer Transformer from "Attention is All You Need". This is almost an exact implementation of the original Transformer encoder. @@ -322,7 +331,8 @@ def transformer_encoder(input_tensor, batch_size=batch_size, from_seq_length=seq_length, to_seq_length=seq_length, - reuse=reuse) + reuse=reuse, + ) # Run a linear projection of `hidden_size` then add a residual # with `layer_input`. @@ -330,7 +340,8 @@ def transformer_encoder(input_tensor, attention_output = tf.layers.dense( attention_output, hidden_size, - kernel_initializer=create_initializer(initializer_range)) + kernel_initializer=create_initializer(initializer_range), + ) attention_output = dropout(attention_output, hidden_dropout_prob) attention_output = layer_norm(attention_output + layer_input) @@ -340,14 +351,16 @@ def transformer_encoder(input_tensor, attention_output, intermediate_size, activation=intermediate_act_fn, - kernel_initializer=create_initializer(initializer_range)) + kernel_initializer=create_initializer(initializer_range), + ) # Down-project back to `hidden_size` then add the residual. with tf.variable_scope('output', reuse=reuse): layer_output = tf.layers.dense( intermediate_output, hidden_size, - kernel_initializer=create_initializer(initializer_range)) + kernel_initializer=create_initializer(initializer_range), + ) layer_output = dropout(layer_output, hidden_dropout_prob) layer_output = layer_norm(layer_output + attention_output) prev_output = layer_output @@ -356,18 +369,20 @@ def transformer_encoder(input_tensor, return final_output -def cross_attention_block(from_tensor, - to_tensor, - layer_idx, - size_per_head, - cross_attention_mask=None, - self_attention_mask=None, - num_attention_heads=1, - intermediate_size=512, - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - initializer_range=0.02, - name=''): +def cross_attention_block( + from_tensor, + to_tensor, + layer_idx, + size_per_head, + cross_attention_mask=None, + self_attention_mask=None, + num_attention_heads=1, + intermediate_size=512, + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + initializer_range=0.02, + name='', +): """Multi-headed cross attention block. Args: @@ -421,7 +436,8 @@ def cross_attention_block(from_tensor, do_return_2d_tensor=True, batch_size=batch_size, from_seq_length=from_seq_length, - to_seq_length=to_seq_length) + to_seq_length=to_seq_length, + ) with tf.variable_scope('self'): # [batch_size * from_seq_length, num_attention_heads * size_per_head] @@ -436,7 +452,8 @@ def cross_attention_block(from_tensor, do_return_2d_tensor=True, batch_size=batch_size, from_seq_length=from_seq_length, - to_seq_length=from_seq_length) + to_seq_length=from_seq_length, + ) with tf.variable_scope('output'): attention_output = dropout(self_attention_output, hidden_dropout_prob) @@ -448,14 +465,16 @@ def cross_attention_block(from_tensor, attention_output, intermediate_size, activation=tf.nn.relu, - kernel_initializer=create_initializer(initializer_range)) + kernel_initializer=create_initializer(initializer_range), + ) # Down-project back to `hidden_size` then add the residual. with tf.variable_scope('output'): layer_output = tf.layers.dense( intermediate_output, num_attention_heads * size_per_head, - kernel_initializer=create_initializer(initializer_range)) + kernel_initializer=create_initializer(initializer_range), + ) layer_output = dropout(layer_output, hidden_dropout_prob) # [batch_size * from_seq_length, num_attention_heads * size_per_head] layer_output = layer_norm(layer_output + attention_output) @@ -466,20 +485,22 @@ def cross_attention_block(from_tensor, return final_output # [batch_size, from_seq_length, num_attention_heads * size_per_head] -def cross_attention_tower(left_tensor, - right_tensor, - num_hidden_layers=1, - num_attention_heads=12, - left_size_per_head=64, - right_size_per_head=64, - left_intermediate_size=0, - right_intermediate_size=0, - left_input_mask=None, - right_input_mask=None, - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - initializer_range=0.02, - name=''): +def cross_attention_tower( + left_tensor, + right_tensor, + num_hidden_layers=1, + num_attention_heads=12, + left_size_per_head=64, + right_size_per_head=64, + left_intermediate_size=0, + right_intermediate_size=0, + left_input_mask=None, + right_input_mask=None, + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + initializer_range=0.02, + name='', +): """Multi-headed, multi layer cross attention block. Args: @@ -553,7 +574,8 @@ def cross_attention_tower(left_tensor, self_attention_mask=left_attention_mask, attention_probs_dropout_prob=attention_probs_dropout_prob, initializer_range=initializer_range, - name='%sleft_to_right_' % name) + name='%sleft_to_right_' % name, + ) right_output = cross_attention_block( prev_right_output, prev_left_output, @@ -566,7 +588,8 @@ def cross_attention_tower(left_tensor, self_attention_mask=right_attention_mask, attention_probs_dropout_prob=attention_probs_dropout_prob, initializer_range=initializer_range, - name='%sright_to_left_' % name) + name='%sright_to_left_' % name, + ) prev_left_output = left_output prev_right_output = right_output return prev_left_output, prev_right_output @@ -639,18 +662,20 @@ def create_attention_mask_from_input_mask(from_tensor, to_mask): return mask -def embedding_postprocessor(input_tensor, - use_token_type=False, - token_type_ids=None, - token_type_vocab_size=16, - token_type_embedding_name='token_type_embeddings', - reuse_token_type=None, - use_position_embeddings=True, - position_embedding_name='position_embeddings', - reuse_position_embedding=None, - initializer_range=0.02, - max_position_embeddings=512, - dropout_prob=0.1): +def embedding_postprocessor( + input_tensor, + use_token_type=False, + token_type_ids=None, + token_type_vocab_size=16, + token_type_embedding_name='token_type_embeddings', + reuse_token_type=None, + use_position_embeddings=True, + position_embedding_name='position_embeddings', + reuse_position_embedding=None, + initializer_range=0.02, + max_position_embeddings=512, + dropout_prob=0.0, +): """Performs various post-processing on a word embedding tensor. Args: @@ -695,7 +720,8 @@ def embedding_postprocessor(input_tensor, token_type_table = tf.get_variable( name=token_type_embedding_name, shape=[token_type_vocab_size, width], - initializer=create_initializer(initializer_range)) + initializer=create_initializer(initializer_range), + ) # This vocab will be small so we always do one-hot here, since it is always # faster for a small vocabulary. flat_token_type_ids = tf.reshape(token_type_ids, [-1]) @@ -713,7 +739,8 @@ def embedding_postprocessor(input_tensor, full_position_embeddings = tf.get_variable( name=position_embedding_name, shape=[max_position_embeddings, width], - initializer=create_initializer(initializer_range)) + initializer=create_initializer(initializer_range), + ) # Since the position embedding table is a learned variable, we create it # using a (long) sequence length `max_position_embeddings`. The actual # sequence length might be shorter than this, for faster training of diff --git a/easy_rec/python/layers/senet.py b/easy_rec/python/layers/senet.py index 777079341..ce85d7ccc 100644 --- a/easy_rec/python/layers/senet.py +++ b/easy_rec/python/layers/senet.py @@ -58,13 +58,15 @@ def __call__(self, inputs): units=reduction_size, kernel_regularizer=self._l2_reg, activation='relu', - name='%s/reduce' % self._name) + name='%s/reduce' % self._name, + ) excited_weights = tf.layers.dense( inputs=reduced, units=emb_size, kernel_initializer='glorot_normal', - name='%s/excite' % self._name) + name='%s/excite' % self._name, + ) # Re-weight inputs = tf.concat(inputs, axis=-1) diff --git a/easy_rec/python/layers/seq_input_layer.py b/easy_rec/python/layers/seq_input_layer.py index a52904dd1..ce3744009 100644 --- a/easy_rec/python/layers/seq_input_layer.py +++ b/easy_rec/python/layers/seq_input_layer.py @@ -18,11 +18,13 @@ class SeqInputLayer(object): - def __init__(self, - feature_configs, - feature_groups_config, - embedding_regularizer=None, - ev_params=None): + def __init__( + self, + feature_configs, + feature_groups_config, + embedding_regularizer=None, + ev_params=None, + ): self._feature_groups_config = { x.group_name: x for x in feature_groups_config } @@ -31,12 +33,14 @@ def __init__(self, feature_configs, wide_and_deep_dict, ev_params=ev_params) self._embedding_regularizer = embedding_regularizer - def __call__(self, - features, - group_name, - feature_name_to_output_tensors={}, - allow_key_search=True, - scope_name=None): + def __call__( + self, + features, + group_name, + feature_name_to_output_tensors={}, + allow_key_search=True, + scope_name=None, + ): feature_column_dict = self._fc_parser.deep_columns feature_column_dict.update(self._fc_parser.sequence_columns) @@ -70,11 +74,14 @@ def _seq_embed_summary_name(input_name): tmp_key_tensor = feature_column_dict[key]._get_dense_tensor( builder) regularizers.apply_regularization( - self._embedding_regularizer, weights_list=[tmp_key_tensor]) + self._embedding_regularizer, + weights_list=[tmp_key_tensor], + ) key_tensors.append(tmp_key_tensor) elif feature_name_to_output_tensors[key] is None: - assert feature_name_to_output_tensors[ - key] is not None, 'When allow_key_search is False, key: %s should defined in same feature group.' % key + assert feature_name_to_output_tensors[key] is not None, ( + 'When allow_key_search is False, key: %s should defined in same feature group.' + % key) else: key_tensors.append(feature_name_to_output_tensors[key]) @@ -111,7 +118,8 @@ def _seq_embed_summary_name(input_name): cur_hist_seqs[0][1], cur_hist_seqs[idx][1], message='SequenceFeature Error: The size of %s not equal to the size of %s.' - % (x.hist_seq[idx], x.hist_seq[0])) + % (x.hist_seq[idx], x.hist_seq[0]), + ) check_op_list.append(check_op) with tf.control_dependencies(check_op_list): @@ -119,7 +127,7 @@ def _seq_embed_summary_name(input_name): 'key': tf.concat(key_tensors, axis=-1), 'hist_seq_emb': tf.concat([x[0] for x in hist_tensors], axis=-1), 'hist_seq_len': hist_tensors[0][1], - 'aux_hist_seq_emb_list': aux_hist_emb_list + 'aux_hist_seq_emb_list': aux_hist_emb_list, } return features diff --git a/easy_rec/python/layers/sequence_feature_layer.py b/easy_rec/python/layers/sequence_feature_layer.py index fd01b5b2c..196f89483 100644 --- a/easy_rec/python/layers/sequence_feature_layer.py +++ b/easy_rec/python/layers/sequence_feature_layer.py @@ -15,14 +15,16 @@ class SequenceFeatureLayer(object): - def __init__(self, - feature_configs, - feature_groups_config, - ev_params=None, - embedding_regularizer=None, - kernel_regularizer=None, - is_training=False, - is_predicting=False): + def __init__( + self, + feature_configs, + feature_groups_config, + ev_params=None, + embedding_regularizer=None, + kernel_regularizer=None, + is_training=False, + is_predicting=False, + ): self._seq_feature_groups_config = [] for x in feature_groups_config: for y in x.sequence_features: @@ -33,22 +35,28 @@ def __init__(self, feature_configs, self._seq_feature_groups_config, embedding_regularizer=embedding_regularizer, - ev_params=ev_params) + ev_params=ev_params, + ) self._embedding_regularizer = embedding_regularizer self._kernel_regularizer = kernel_regularizer self._is_training = is_training self._is_predicting = is_predicting - def negative_sampler_target_attention(self, - dnn_config, - deep_fea, - concat_features, - name, - need_key_feature=True, - allow_key_transform=False): - cur_id, hist_id_col, seq_len, aux_hist_emb_list = deep_fea['key'], deep_fea[ - 'hist_seq_emb'], deep_fea['hist_seq_len'], deep_fea[ - 'aux_hist_seq_emb_list'] + def negative_sampler_target_attention( + self, + dnn_config, + deep_fea, + concat_features, + name, + need_key_feature=True, + allow_key_transform=False, + ): + cur_id, hist_id_col, seq_len, aux_hist_emb_list = ( + deep_fea['key'], + deep_fea['hist_seq_emb'], + deep_fea['hist_seq_len'], + deep_fea['aux_hist_seq_emb_list'], + ) seq_max_len = tf.shape(hist_id_col)[1] seq_emb_dim = hist_id_col.shape[2] @@ -57,11 +65,14 @@ def negative_sampler_target_attention(self, pos_feature = cur_id[:batch_size] neg_feature = cur_id[batch_size:] - cur_id = tf.concat([ - pos_feature[:, tf.newaxis, :], - tf.tile(neg_feature[tf.newaxis, :, :], multiples=[batch_size, 1, 1]) - ], - axis=1) # noqa: E126 + cur_id = tf.concat( + [ + pos_feature[:, tf.newaxis, :], + tf.tile( + neg_feature[tf.newaxis, :, :], multiples=[batch_size, 1, 1]), + ], + axis=1, + ) # noqa: E126 neg_num_add_1 = tf.shape(cur_id)[1] hist_id_col_tmp = tf.tile( hist_id_col[:, :, :], multiples=[1, neg_num_add_1, 1]) @@ -83,7 +94,8 @@ def negative_sampler_target_attention(self, din_net = tf.concat( [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], - axis=-1) # (B * neg_num_add_1, seq_max_len, seq_emb_dim*4) + axis=-1, + ) # (B * neg_num_add_1, seq_max_len, seq_emb_dim*4) din_layer = dnn.DNN( dnn_config, @@ -91,13 +103,14 @@ def negative_sampler_target_attention(self, name, self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True) + last_layer_no_batch_norm=True, + ) din_net = din_layer(din_net) scores = tf.reshape(din_net, [-1, 1, seq_max_len]) # (B, 1, ?) seq_len = tf.expand_dims(seq_len, 1) mask = tf.sequence_mask(seq_len) - padding = tf.ones_like(scores) * (-2**32 + 1) + padding = tf.ones_like(scores) * (-(2**32) + 1) scores = tf.where(mask, scores, padding) # [B*neg_num_add_1, 1, seq_max_len] @@ -120,16 +133,21 @@ def negative_sampler_target_attention(self, din_output = tf.concat([hist_din_emb, cur_id], axis=2) return din_output, concat_features - def target_attention(self, - dnn_config, - deep_fea, - name, - need_key_feature=True, - allow_key_transform=False, - transform_dnn=False): - cur_id, hist_id_col, seq_len, aux_hist_emb_list = deep_fea['key'], deep_fea[ - 'hist_seq_emb'], deep_fea['hist_seq_len'], deep_fea[ - 'aux_hist_seq_emb_list'] + def target_attention( + self, + dnn_config, + deep_fea, + name, + need_key_feature=True, + allow_key_transform=False, + transform_dnn=False, + ): + cur_id, hist_id_col, seq_len, aux_hist_emb_list = ( + deep_fea['key'], + deep_fea['hist_seq_emb'], + deep_fea['hist_seq_len'], + deep_fea['aux_hist_seq_emb_list'], + ) seq_max_len = tf.shape(hist_id_col)[1] seq_emb_dim = hist_id_col.shape[2] @@ -153,7 +171,8 @@ def target_attention(self, din_net = tf.concat( [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], - axis=-1) # (B, seq_max_len, seq_emb_dim*4) + axis=-1, + ) # (B, seq_max_len, seq_emb_dim*4) din_layer = dnn.DNN( dnn_config, @@ -161,13 +180,14 @@ def target_attention(self, name, self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True) + last_layer_no_batch_norm=True, + ) din_net = din_layer(din_net) scores = tf.reshape(din_net, [-1, 1, seq_max_len]) # (B, 1, ?) seq_len = tf.expand_dims(seq_len, 1) mask = tf.sequence_mask(seq_len) - padding = tf.ones_like(scores) * (-2**32 + 1) + padding = tf.ones_like(scores) * (-(2**32) + 1) scores = tf.where(mask, scores, padding) # [B, 1, seq_max_len] # Scale @@ -188,13 +208,15 @@ def target_attention(self, din_output = tf.concat([hist_din_emb, cur_id], axis=1) return din_output - def __call__(self, - features, - concat_features, - all_seq_att_map_config, - feature_name_to_output_tensors=None, - negative_sampler=False, - scope_name=None): + def __call__( + self, + features, + concat_features, + all_seq_att_map_config, + feature_name_to_output_tensors=None, + negative_sampler=False, + scope_name=None, + ): logging.info('use sequence feature layer.') all_seq_fea = [] # process all sequence features @@ -209,9 +231,13 @@ def __call__(self, place_on_cpu = eval(place_on_cpu) if place_on_cpu else False with conditional(self._is_predicting and place_on_cpu, ops.device('/CPU:0')): - seq_features = self._seq_input_layer(features, group_name, - feature_name_to_output_tensors, - allow_key_search, scope_name) + seq_features = self._seq_input_layer( + features, + group_name, + feature_name_to_output_tensors, + allow_key_search, + scope_name, + ) # apply regularization for sequence feature key in seq_input_layer. @@ -226,6 +252,7 @@ def __call__(self, 'seq_dnn not set in seq_att_groups, will use default settings') # If not set seq_dnn, will use default settings from easy_rec.python.protos.dnn_pb2 import DNN + seq_dnn_config = DNN() seq_dnn_config.hidden_units.extend([128, 64, 32, 1]) cur_target_attention_name = 'seq_dnn' + group_name @@ -236,7 +263,8 @@ def __call__(self, concat_features, name=cur_target_attention_name, need_key_feature=need_key_feature, - allow_key_transform=allow_key_transform) + allow_key_transform=allow_key_transform, + ) else: seq_fea = self.target_attention( seq_dnn_config, @@ -244,6 +272,7 @@ def __call__(self, name=cur_target_attention_name, need_key_feature=need_key_feature, allow_key_transform=allow_key_transform, - transform_dnn=transform_dnn) + transform_dnn=transform_dnn, + ) all_seq_fea.append(seq_fea) return concat_features, all_seq_fea diff --git a/easy_rec/python/layers/uniter.py b/easy_rec/python/layers/uniter.py index 3018bad61..35fbd2e1b 100644 --- a/easy_rec/python/layers/uniter.py +++ b/easy_rec/python/layers/uniter.py @@ -50,18 +50,21 @@ def __init__(self, model_config, feature_configs, features, uniter_config, if fea_group.group_name == 'general': self._general_feature_num = len(fea_group.feature_names) general_feature_names = set(fea_group.feature_names) - assert self._general_feature_num == len(general_feature_names), ( - 'there are duplicate features in `general` feature group') + assert self._general_feature_num == len( + general_feature_names + ), 'there are duplicate features in `general` feature group' elif fea_group.group_name == 'image': self._img_feature_num = len(fea_group.feature_names) img_feature_names = set(fea_group.feature_names) - assert self._img_feature_num == len(img_feature_names), ( - 'there are duplicate features in `image` feature group') + assert self._img_feature_num == len( + img_feature_names + ), 'there are duplicate features in `image` feature group' elif fea_group.group_name == 'text': self._txt_feature_num = len(fea_group.feature_names) txt_feature_names = set(fea_group.feature_names) - assert self._txt_feature_num == len(txt_feature_names), ( - 'there are duplicate features in `text` feature group') + assert self._txt_feature_num == len( + txt_feature_names + ), 'there are duplicate features in `text` feature group' if self._txt_feature_num > 1 or self._img_feature_num > 1: self._use_token_type = True @@ -93,30 +96,29 @@ def __init__(self, model_config, feature_configs, features, uniter_config, max_seq_len = feature_config.max_seq_len unique_dim_num = len(set(txt_fea_emb_dim_list)) - assert unique_dim_num <= 1 and len( - txt_fea_emb_dim_list - ) == self._txt_feature_num, ( - 'Uniter requires that all `text` feature dimensions must be consistent.' - ) + assert ( + unique_dim_num <= 1 and + len(txt_fea_emb_dim_list) == self._txt_feature_num + ), 'Uniter requires that all `text` feature dimensions must be consistent.' unique_dim_num = len(set(img_fea_emb_dim_list)) - assert unique_dim_num <= 1 and len( - img_fea_emb_dim_list - ) == self._img_feature_num, ( - 'Uniter requires that all `image` feature dimensions must be consistent.' - ) + assert ( + unique_dim_num <= 1 and + len(img_fea_emb_dim_list) == self._img_feature_num + ), 'Uniter requires that all `image` feature dimensions must be consistent.' unique_dim_num = len(set(general_emb_dim_list)) - assert unique_dim_num <= 1 and len( - general_emb_dim_list - ) == self._general_feature_num, ( - 'Uniter requires that all `general` feature dimensions must be consistent.' - ) + assert ( + unique_dim_num <= 1 and + len(general_emb_dim_list) == self._general_feature_num + ), 'Uniter requires that all `general` feature dimensions must be consistent.' if self._txt_feature_num > 0 and uniter_config.use_position_embeddings: - assert uniter_config.max_position_embeddings > 0, ( - 'model config `max_position_embeddings` must be greater than 0. ') + assert ( + uniter_config.max_position_embeddings > + 0), 'model config `max_position_embeddings` must be greater than 0. ' assert uniter_config.max_position_embeddings >= max_seq_len, ( - 'model config `max_position_embeddings` must be greater than or equal to the maximum of all feature config ' - '`max_seq_len`, which is %d' % max_seq_len) + 'model config `max_position_embeddings` must be greater than' + ' or equal to the maximum of all feature config ' + '`max_seq_len`, which is %d') % max_seq_len self._img_emb_size = img_fea_emb_dim_list[0] if img_fea_emb_dim_list else 0 self._txt_emb_size = txt_fea_emb_dim_list[0] if txt_fea_emb_dim_list else 0 @@ -146,11 +148,13 @@ def text_embeddings(self, token_type_id): use_token_type=self._use_token_type, token_type_ids=tf.ones( shape=tf.stack([batch_size, self._general_feature_num]), - dtype=tf.int32) * token_type_id, + dtype=tf.int32, + ) * token_type_id, token_type_vocab_size=self._token_type_vocab_size, reuse_token_type=tf.AUTO_REUSE, use_position_embeddings=False, - dropout_prob=self._model_config.hidden_dropout_prob) + dropout_prob=self._model_config.hidden_dropout_prob, + ) all_txt_features.append(general_features) mask = tf.ones( @@ -185,12 +189,14 @@ def dynamic_mask(x, max_len): use_position_embeddings=self._model_config.use_position_embeddings, max_position_embeddings=self._model_config.max_position_embeddings, position_embedding_name='txt_position_embeddings_%d' % i, - dropout_prob=self._model_config.hidden_dropout_prob) + dropout_prob=self._model_config.hidden_dropout_prob, + ) all_txt_features.append(seq_fea) input_mask = tf.map_fn( fn=lambda t: dynamic_mask(t, max_seq_len), - elems=tf.to_int32(seq_len)) + elems=tf.to_int32(seq_len), + ) input_masks.append(input_mask) return all_txt_features, input_masks @@ -221,7 +227,8 @@ def image_embeddings(self): use_position_embeddings=self._model_config.use_position_embeddings, max_position_embeddings=self._model_config.max_position_embeddings, position_embedding_name='img_position_embeddings', - dropout_prob=self._model_config.hidden_dropout_prob) + dropout_prob=self._model_config.hidden_dropout_prob, + ) return img_fea def __call__(self, is_training, *args, **kwargs): @@ -280,7 +287,8 @@ def __call__(self, is_training, *args, **kwargs): attention_probs_dropout_prob=self._model_config .attention_probs_dropout_prob, initializer_range=self._model_config.initializer_range, - name='uniter') # shape: [batch_size, seq_length, hidden_size] + name='uniter', + ) # shape: [batch_size, seq_length, hidden_size] print('attention_fea:', attention_fea.shape) mm_fea = attention_fea[:, 0, :] # [CLS] feature sub_modules.append(mm_fea) @@ -288,8 +296,12 @@ def __call__(self, is_training, *args, **kwargs): if self._other_features is not None: if self._model_config.HasField('other_feature_dnn'): l2_reg = kwargs['l2_reg'] if 'l2_reg' in kwargs else 0 - other_dnn_layer = dnn.DNN(self._model_config.other_feature_dnn, l2_reg, - 'other_dnn', is_training) + other_dnn_layer = dnn.DNN( + self._model_config.other_feature_dnn, + l2_reg, + 'other_dnn', + is_training, + ) other_fea = other_dnn_layer(self._other_features) else: other_fea = self._other_features diff --git a/easy_rec/python/layers/utils.py b/easy_rec/python/layers/utils.py index 7eb86b791..e650809fd 100644 --- a/easy_rec/python/layers/utils.py +++ b/easy_rec/python/layers/utils.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================== """Common util functions used by layers.""" + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -37,7 +38,7 @@ def _tensor_to_map(tensor): return { 'node_path': tensor.name, 'shape': tensor.shape.as_list() if tensor.shape else None, - 'dtype': tensor.dtype.name + 'dtype': tensor.dtype.name, } @@ -141,8 +142,8 @@ def gen_embedding_attrs(column=None, attrs['is_embedding_var'] = True attrs['embedding_var_keys'] = variable._shared_name + '-keys' attrs['embedding_var_values'] = variable._shared_name + '-values' - elif (isinstance(variable, variables.PartitionedVariable)) and \ - (isinstance(variable._get_variable_list()[0], kv_variable_ops.EmbeddingVariable)): + elif (isinstance(variable, variables.PartitionedVariable)) and (isinstance( + variable._get_variable_list()[0], kv_variable_ops.EmbeddingVariable)): attrs['embedding_var_keys'] = [v._shared_name + '-keys' for v in variable] attrs['embedding_var_values'] = [ v._shared_name + '-values' for v in variable @@ -155,11 +156,13 @@ def gen_embedding_attrs(column=None, def mark_input_src(name, src_desc): - ops.add_to_collection(ops.GraphKeys.RANK_SERVICE_INPUT_SRC, - json.dumps({ - 'name': name, - 'src': src_desc - })) + ops.add_to_collection( + ops.GraphKeys.RANK_SERVICE_INPUT_SRC, + json.dumps({ + 'name': name, + 'src': src_desc + }), + ) def is_proto_message(pb_obj, field): diff --git a/easy_rec/python/layers/variational_dropout_layer.py b/easy_rec/python/layers/variational_dropout_layer.py index 0eeddcf7b..270e3a67a 100644 --- a/easy_rec/python/layers/variational_dropout_layer.py +++ b/easy_rec/python/layers/variational_dropout_layer.py @@ -5,8 +5,10 @@ import numpy as np import tensorflow as tf -from easy_rec.python.compat.feature_column.feature_column import _SharedEmbeddingColumn # NOQA -from easy_rec.python.compat.feature_column.feature_column_v2 import EmbeddingColumn # NOQA +from easy_rec.python.compat.feature_column.feature_column import ( # NOQA + _SharedEmbeddingColumn,) +from easy_rec.python.compat.feature_column.feature_column_v2 import ( # NOQA + EmbeddingColumn,) if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -42,10 +44,12 @@ def __init__(self, name=logit_p_name, shape=self.drop_param_shape, dtype=tf.float32, - initializer=None) + initializer=None, + ) tf.add_to_collection( 'variational_dropout', - json.dumps([name, list(self.features_dimension.items())])) + json.dumps([name, list(self.features_dimension.items())]), + ) def get_lambda(self): return self._config.regularization_lambda @@ -110,8 +114,8 @@ def concrete_dropout_neuron(self, dropout_p, temp=1.0 / 10.0): tf.shape(dropout_p), dtype=tf.float32, seed=None, name='unif_noise') approx = ( - tf.log(dropout_p + EPSILON) - tf.log(1. - dropout_p + EPSILON) + - tf.log(unif_noise + EPSILON) - tf.log(1. - unif_noise + EPSILON)) + tf.log(dropout_p + EPSILON) - tf.log(1.0 - dropout_p + EPSILON) + + tf.log(unif_noise + EPSILON) - tf.log(1.0 - unif_noise + EPSILON)) approx_output = tf.sigmoid(approx / temp) return 1 - approx_output @@ -120,7 +124,7 @@ def __call__(self, output_features): batch_size = tf.shape(output_features)[0] noisy_input = self.sample_noisy_input(output_features) dropout_p = tf.sigmoid(self.logit_p) - variational_dropout_penalty = 1. - dropout_p + variational_dropout_penalty = 1.0 - dropout_p variational_dropout_penalty_lambda = self.get_lambda() / tf.cast( batch_size, dtype=tf.float32) variational_dropout_loss_sum = variational_dropout_penalty_lambda * tf.reduce_sum( diff --git a/easy_rec/python/loss/focal_loss.py b/easy_rec/python/loss/focal_loss.py index 9ef6a94a7..5b681be95 100644 --- a/easy_rec/python/loss/focal_loss.py +++ b/easy_rec/python/loss/focal_loss.py @@ -8,14 +8,16 @@ tf = tf.compat.v1 -def sigmoid_focal_loss_with_logits(labels, - logits, - gamma=2.0, - alpha=None, - ohem_ratio=1.0, - sample_weights=None, - label_smoothing=0, - name=''): +def sigmoid_focal_loss_with_logits( + labels, + logits, + gamma=2.0, + alpha=None, + ohem_ratio=1.0, + sample_weights=None, + label_smoothing=0, + name='', +): """Implements the focal loss function. Focal loss was first introduced in the RetinaNet paper @@ -85,7 +87,8 @@ def sigmoid_focal_loss_with_logits(labels, logits, weights=weights, label_smoothing=label_smoothing, - reduction=tf.losses.Reduction.NONE) + reduction=tf.losses.Reduction.NONE, + ) k = tf.to_float(tf.size(losses)) * tf.convert_to_tensor(ohem_ratio) k = tf.to_int32(tf.math.rint(k)) topk = tf.nn.top_k(losses, k) diff --git a/easy_rec/python/loss/jrc_loss.py b/easy_rec/python/loss/jrc_loss.py index b5165d3c2..6d6c27059 100644 --- a/easy_rec/python/loss/jrc_loss.py +++ b/easy_rec/python/loss/jrc_loss.py @@ -9,14 +9,16 @@ tf = tf.compat.v1 -def jrc_loss(labels, - logits, - session_ids, - alpha=0.5, - loss_weight_strategy='fixed', - sample_weights=1.0, - same_label_loss=True, - name=''): +def jrc_loss( + labels, + logits, + session_ids, + alpha=0.5, + loss_weight_strategy='fixed', + sample_weights=1.0, + same_label_loss=True, + name='', +): """Joint Optimization of Ranking and Calibration with Contextualized Hybrid Model. https://arxiv.org/abs/2208.06164 @@ -105,8 +107,10 @@ def jrc_loss(labels, bern = tf.distributions.Bernoulli(probs=0.5, dtype=tf.float32) weights = bern.sample(2) loss_weight = tf.cond( - tf.equal(tf.reduce_sum(weights), 1), lambda: weights, - lambda: tf.convert_to_tensor([0.5, 0.5])) + tf.equal(tf.reduce_sum(weights), 1), + lambda: weights, + lambda: tf.convert_to_tensor([0.5, 0.5]), + ) loss = loss_weight[0] * ce_loss + loss_weight[1] * ge_loss tf.summary.scalar('loss/%s_ce_weight' % loss_name, loss_weight[0]) tf.summary.scalar('loss/%s_ge_weight' % loss_name, loss_weight[1]) diff --git a/easy_rec/python/loss/listwise_loss.py b/easy_rec/python/loss/listwise_loss.py index 24bd5864f..39c8e898c 100644 --- a/easy_rec/python/loss/listwise_loss.py +++ b/easy_rec/python/loss/listwise_loss.py @@ -25,15 +25,17 @@ def _list_prob_loss(x, labels, logits, session_ids): return -tf.reduce_sum(y * y_hat) -def listwise_rank_loss(labels, - logits, - session_ids, - transform_fn=None, - temperature=1.0, - label_is_logits=False, - scale_logits=False, - weights=1.0, - name='listwise_loss'): +def listwise_rank_loss( + labels, + logits, + session_ids, + transform_fn=None, + temperature=1.0, + label_is_logits=False, + scale_logits=False, + weights=1.0, + name='listwise_loss', +): r"""Computes listwise softmax cross entropy loss between `labels` and `logits`. Definition: @@ -65,12 +67,14 @@ def listwise_rank_loss(labels, 'scale_w', dtype=tf.float32, shape=(1,), - initializer=tf.ones_initializer()) + initializer=tf.ones_initializer(), + ) b = tf.get_variable( 'scale_b', dtype=tf.float32, shape=(1,), - initializer=tf.zeros_initializer()) + initializer=tf.zeros_initializer(), + ) logits = logits * tf.abs(w) + b if temperature != 1.0: logits /= temperature @@ -86,7 +90,8 @@ def listwise_rank_loss(labels, lambda x: _list_wise_loss(x, labels, logits, session_ids, label_is_logits ), sessions, - dtype=tf.float32) + dtype=tf.float32, + ) if tf.is_numeric_tensor(weights): logging.error('[%s] use unsupported sample weight' % loss_name) return tf.reduce_mean(losses) @@ -94,15 +99,17 @@ def listwise_rank_loss(labels, return tf.reduce_mean(losses) * weights -def listwise_distill_loss(labels, - logits, - session_ids, - transform_fn=None, - temperature=1.0, - label_clip_max_value=512, - scale_logits=False, - weights=1.0, - name='listwise_distill_loss'): +def listwise_distill_loss( + labels, + logits, + session_ids, + transform_fn=None, + temperature=1.0, + label_clip_max_value=512, + scale_logits=False, + weights=1.0, + name='listwise_distill_loss', +): r"""Computes listwise softmax cross entropy loss between `labels` and `logits`. Definition: @@ -138,12 +145,14 @@ def listwise_distill_loss(labels, 'scale_w', dtype=tf.float32, shape=(1,), - initializer=tf.ones_initializer()) + initializer=tf.ones_initializer(), + ) b = tf.get_variable( 'scale_b', dtype=tf.float32, shape=(1,), - initializer=tf.zeros_initializer()) + initializer=tf.zeros_initializer(), + ) logits = logits * tf.abs(w) + b if temperature != 1.0: logits /= temperature @@ -153,7 +162,8 @@ def listwise_distill_loss(labels, losses = tf.map_fn( lambda x: _list_prob_loss(x, labels, logits, session_ids), sessions, - dtype=tf.float32) + dtype=tf.float32, + ) if tf.is_numeric_tensor(weights): logging.error('[%s] use unsupported sample weight' % loss_name) return tf.reduce_mean(losses) diff --git a/easy_rec/python/loss/multi_similarity.py b/easy_rec/python/loss/multi_similarity.py index 2b7d4e0e9..04a863bee 100644 --- a/easy_rec/python/loss/multi_similarity.py +++ b/easy_rec/python/loss/multi_similarity.py @@ -8,15 +8,17 @@ tf = tf.compat.v1 -def ms_loss(embeddings, - labels, - session_ids=None, - alpha=2.0, - beta=50.0, - lamb=1.0, - eps=0.1, - ms_mining=False, - embed_normed=False): +def ms_loss( + embeddings, + labels, + session_ids=None, + alpha=2.0, + beta=50.0, + lamb=1.0, + eps=0.1, + ms_mining=False, + embed_normed=False, +): """Refer paper: Multi-Similarity Loss with General Pair Weighting for Deep Metric Learning. ref: http://openaccess.thecvf.com/content_CVPR_2019/papers/ diff --git a/easy_rec/python/loss/softmax_loss_with_negative_mining.py b/easy_rec/python/loss/softmax_loss_with_negative_mining.py index 99f92d4af..643b84689 100644 --- a/easy_rec/python/loss/softmax_loss_with_negative_mining.py +++ b/easy_rec/python/loss/softmax_loss_with_negative_mining.py @@ -30,16 +30,18 @@ def support_vector_guided_softmax_loss(pos_score, return loss -def softmax_loss_with_negative_mining(user_emb, - item_emb, - labels, - num_negative_samples=4, - embed_normed=False, - weights=1.0, - gamma=1.0, - margin=0, - t=1, - seed=None): +def softmax_loss_with_negative_mining( + user_emb, + item_emb, + labels, + num_negative_samples=4, + embed_normed=False, + weights=1.0, + gamma=1.0, + margin=0, + t=1, + seed=None, +): """Compute the softmax loss based on the cosine distance explained below. Given mini batches for `user_emb` and `item_emb`, this function computes for each element in `user_emb` @@ -74,7 +76,8 @@ def softmax_loss_with_negative_mining(user_emb, is_valid = tf.assert_less( num_negative_samples, batch_size, - message='`num_negative_samples` should be less than batch_size') + message='`num_negative_samples` should be less than batch_size', + ) with tf.control_dependencies([is_valid]): if not embed_normed: user_emb = tf.nn.l2_normalize(user_emb, axis=-1) diff --git a/easy_rec/python/loss/zero_inflated_lognormal.py b/easy_rec/python/loss/zero_inflated_lognormal.py index e3ae3110e..3ffb9a3b4 100644 --- a/easy_rec/python/loss/zero_inflated_lognormal.py +++ b/easy_rec/python/loss/zero_inflated_lognormal.py @@ -1,6 +1,9 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Zero-inflated lognormal loss for lifetime value prediction.""" + +import logging + import tensorflow as tf import tensorflow_probability as tfp @@ -10,27 +13,55 @@ tf = tf.compat.v1 -def zero_inflated_lognormal_pred(logits): +def log_sigmoid(x): + return -tf.nn.softplus(-x) # 兼容 TF 1.12 + + +def zero_inflated_lognormal_pred(logits, + max_sigma=5.0, + max_log_clip=20.0, + return_log=False): """Calculates predicted mean of zero inflated lognormal logits. Arguments: logits: [batch_size, 3] tensor of logits. + max_sigma: max value of sigma + return_log: whether return log value + max_log_clip: max clip value of log space Returns: positive_probs: [batch_size, 1] tensor of positive probability. preds: [batch_size, 1] tensor of predicted mean. """ + logging.info('max_sigma=%f, max_log_clip=%f' % (max_sigma, max_log_clip)) logits = tf.convert_to_tensor(logits, dtype=tf.float32) positive_probs = tf.keras.backend.sigmoid(logits[..., :1]) - loc = logits[..., 1:2] - scale = tf.keras.backend.softplus(logits[..., 2:]) - preds = ( - positive_probs * - tf.keras.backend.exp(loc + 0.5 * tf.keras.backend.square(scale))) - return positive_probs, preds + log_positive_probs = log_sigmoid(logits[..., :1]) + mu = logits[..., 1:2] + sigma = tf.keras.backend.softplus(logits[..., 2:]) + sigma = tf.clip_by_value(sigma, tf.math.sqrt(tf.keras.backend.epsilon()), + max_sigma) + log_mean_pos = mu + 0.5 * tf.keras.backend.square(sigma) + log_preds = log_positive_probs + log_mean_pos + if return_log: + logging.info('return_log=true') + return positive_probs, log_preds + else: + log_preds = tf.clip_by_value(log_preds, -100.0, max_log_clip) + preds = tf.keras.backend.exp(log_preds) + return positive_probs, preds -def zero_inflated_lognormal_loss(labels, logits, name=''): +def zero_inflated_lognormal_loss( + labels, + logits, + max_sigma=5.0, + mu_reg=0.01, + sigma_reg=0.01, + class_weight=1.0, + reg_weight=1.0, + name='', +): """Computes the zero inflated lognormal loss. Usage with tf.keras API: @@ -43,12 +74,20 @@ def zero_inflated_lognormal_loss(labels, logits, name=''): Arguments: labels: True targets, tensor of shape [batch_size, 1]. logits: Logits of output layer, tensor of shape [batch_size, 3]. + max_sigma: max value of sigma + mu_reg: regularize coefficient of mu + sigma_reg: regularize coefficient of sigma + class_weight: weight of classification loss + reg_weight: weight of regression loss name: the name of loss Returns: Zero inflated lognormal loss value. """ loss_name = name if name else 'ziln_loss' + logging.info( + '%s max_sigma=%f, mu_reg=%f, sigma_reg=%f, classify weight:%f, regression weight %f' + % (loss_name, max_sigma, mu_reg, sigma_reg, class_weight, reg_weight)) labels = tf.cast(labels, dtype=tf.float32) if labels.shape.ndims == 1: labels = tf.expand_dims(labels, 1) # [B, 1] @@ -64,13 +103,23 @@ def zero_inflated_lognormal_loss(labels, logits, name=''): classification_loss = tf.keras.backend.mean(classification_loss) tf.summary.scalar('loss/%s_classify' % loss_name, classification_loss) - loc = logits[..., 1:2] - scale = tf.math.maximum( - tf.keras.backend.softplus(logits[..., 2:]), - tf.math.sqrt(tf.keras.backend.epsilon())) + mu = logits[..., 1:2] + sigma = tf.keras.backend.softplus(logits[..., 2:]) + sigma = tf.clip_by_value(sigma, tf.math.sqrt(tf.keras.backend.epsilon()), + max_sigma) + safe_labels = positive * labels + ( 1 - positive) * tf.keras.backend.ones_like(labels) - regression_loss = -tf.keras.backend.mean( - positive * tfd.LogNormal(loc=loc, scale=scale).log_prob(safe_labels)) + logprob = tfd.LogNormal(loc=mu, scale=sigma).log_prob(safe_labels) + num_pos = tf.reduce_sum(positive) + 1e-8 + regression_loss = -(tf.reduce_sum(positive * logprob) / num_pos) tf.summary.scalar('loss/%s_regression' % loss_name, regression_loss) - return classification_loss + regression_loss + + # add regular terms + loc_penalty = mu_reg * tf.reduce_mean(tf.square(mu)) + scale_penalty = sigma_reg * tf.reduce_mean(tf.square(logits[..., 2:])) + tf.summary.scalar('loss/%s_mu_penalty' % loss_name, loc_penalty) + tf.summary.scalar('loss/%s_sigma_penalty' % loss_name, scale_penalty) + + total_loss = class_weight * classification_loss + reg_weight * regression_loss + return total_loss + loc_penalty + scale_penalty diff --git a/easy_rec/python/main.py b/easy_rec/python/main.py index 1747155fe..325dabccb 100644 --- a/easy_rec/python/main.py +++ b/easy_rec/python/main.py @@ -23,17 +23,17 @@ from easy_rec.python.model.easy_rec_estimator import EasyRecEstimator from easy_rec.python.model.easy_rec_model import EasyRecModel from easy_rec.python.protos.train_pb2 import DistributionStrategy -from easy_rec.python.utils import config_util -from easy_rec.python.utils import constant -from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils import fg_util -from easy_rec.python.utils import load_class -from easy_rec.python.utils.config_util import get_eval_input_path -from easy_rec.python.utils.config_util import get_model_dir_path -from easy_rec.python.utils.config_util import get_train_input_path -from easy_rec.python.utils.config_util import set_eval_input_path -from easy_rec.python.utils.export_big_model import export_big_model -from easy_rec.python.utils.export_big_model import export_big_model_to_oss + +from easy_rec.python.utils import ( # NOQA + config_util, constant, estimator_utils, fg_util, load_class, +) +from easy_rec.python.utils.config_util import ( # NOQA + get_eval_input_path, get_model_dir_path, get_train_input_path, + set_eval_input_path, +) +from easy_rec.python.utils.export_big_model import ( # NOQA + export_big_model, export_big_model_to_oss, +) try: import horovod.tensorflow as hvd @@ -64,12 +64,14 @@ BestExporter = exporter.BestExporter -def _get_input_fn(data_config, - feature_configs, - data_path=None, - export_config=None, - check_mode=False, - **kwargs): +def _get_input_fn( + data_config, + feature_configs, + data_path=None, + export_config=None, + check_mode=False, + **kwargs, +): """Build estimator input function. Args: @@ -94,7 +96,8 @@ def _get_input_fn(data_config, task_index=task_id, task_num=task_num, check_mode=check_mode, - **kwargs) + **kwargs, + ) input_fn = input_obj.create_input(export_config) return input_fn @@ -104,15 +107,16 @@ def _create_estimator(pipeline_config, distribution=None, params={}): train_config = pipeline_config.train_config gpu_options = GPUOptions(allow_growth=True) # False) - logging.info( - 'train_config.train_distribute=%s[value=%d]' % - (DistributionStrategy.Name(pipeline_config.train_config.train_distribute), - pipeline_config.train_config.train_distribute)) + logging.info('train_config.train_distribute=%s[value=%d]' % ( + DistributionStrategy.Name(pipeline_config.train_config.train_distribute), + pipeline_config.train_config.train_distribute, + )) # set gpu options only under hvd scenes if hvd is not None and pipeline_config.train_config.train_distribute in [ DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy, DistributionStrategy.HorovodStrategy + DistributionStrategy.SokStrategy, + DistributionStrategy.HorovodStrategy, ]: local_rnk = hvd.local_rank() gpus = tf.config.experimental.list_physical_devices('GPU') @@ -126,12 +130,13 @@ def _create_estimator(pipeline_config, distribution=None, params={}): allow_soft_placement=True, log_device_placement=params.get('log_device_placement', False), inter_op_parallelism_threads=train_config.inter_op_parallelism_threads, - intra_op_parallelism_threads=train_config.intra_op_parallelism_threads) + intra_op_parallelism_threads=train_config.intra_op_parallelism_threads, + ) if constant.NO_ARITHMETRIC_OPTI in os.environ: logging.info('arithmetic_optimization is closed to improve performance') - session_config.graph_options.rewrite_options.arithmetic_optimization = \ - session_config.graph_options.rewrite_options.OFF + session_config.graph_options.rewrite_options.arithmetic_optimization = ( + session_config.graph_options.rewrite_options.OFF) session_config.device_filters.append('/job:ps') model_cls = EasyRecModel.create_class(model_config.model_class) @@ -156,7 +161,8 @@ def _create_estimator(pipeline_config, distribution=None, params={}): keep_checkpoint_max=train_config.keep_checkpoint_max, train_distribute=distribution, eval_distribute=distribution, - session_config=session_config) + session_config=session_config, + ) estimator = EasyRecEstimator( pipeline_config, model_cls, run_config=run_config, params=params) @@ -185,7 +191,8 @@ def _create_eval_export_spec(pipeline_config, eval_data, check_mode=False): None, export_config, check_mode=check_mode, - **input_fn_kwargs) + **input_fn_kwargs, + ) if export_config.exporter_type == 'final': exporters = [ FinalExporter(name='final', serving_input_receiver_fn=export_input_fn) @@ -195,7 +202,8 @@ def _create_eval_export_spec(pipeline_config, eval_data, check_mode=False): LatestExporter( name='latest', serving_input_receiver_fn=export_input_fn, - exports_to_keep=export_config.exports_to_keep) + exports_to_keep=export_config.exports_to_keep, + ) ] elif export_config.exporter_type == 'best': logging.info( @@ -217,7 +225,8 @@ def _metric_cmp_fn(best_eval_result, current_eval_result): name='best', serving_input_receiver_fn=export_input_fn, compare_fn=_metric_cmp_fn, - exports_to_keep=export_config.exports_to_keep) + exports_to_keep=export_config.exports_to_keep, + ) ] elif export_config.exporter_type == 'none': exporters = [] @@ -233,7 +242,8 @@ def _metric_cmp_fn(best_eval_result, current_eval_result): input_fn=eval_input_fn, steps=eval_steps, throttle_secs=10, - exporters=exporters) + exporters=exporters, + ) return eval_spec @@ -242,11 +252,11 @@ def _check_model_dir(model_dir, continue_train): if not tf.gfile.IsDirectory(model_dir): tf.gfile.MakeDirs(model_dir) else: - assert len(tf.gfile.Glob(model_dir + '/model.ckpt-*.meta')) == 0, \ - 'model_dir[=%s] already exists and not empty(if you ' \ - 'want to continue train on current model_dir please ' \ - 'delete dir %s or specify --continue_train[internal use only])' % ( - model_dir, model_dir) + assert len(tf.gfile.Glob(model_dir + '/model.ckpt-*.meta')) == 0, ( + 'model_dir[=%s] already exists and not empty(if you ' + 'want to continue train on current model_dir please ' + 'delete dir %s or specify --continue_train[internal use only])' % + (model_dir, model_dir)) else: if not tf.gfile.IsDirectory(model_dir): logging.info('%s does not exists, create it automatically' % model_dir) @@ -265,8 +275,7 @@ def _get_ckpt_path(pipeline_config, checkpoint_path): 'will use latest checkpoint %s from %s' % (ckpt_path, pipeline_config.model_dir)) else: - assert False, 'pipeline_config.model_dir(%s) does not exist' \ - % pipeline_config.model_dir + assert False, 'pipeline_config.model_dir(%s) does not exist' % pipeline_config.model_dir return ckpt_path @@ -293,17 +302,18 @@ def train_and_evaluate(pipeline_config_path, continue_train=False): return pipeline_config -def _train_and_evaluate_impl(pipeline_config, - continue_train=False, - check_mode=False, - fit_on_eval=False, - fit_on_eval_steps=None): +def _train_and_evaluate_impl( + pipeline_config, + continue_train=False, + check_mode=False, + fit_on_eval=False, + fit_on_eval_steps=None, +): train_config = pipeline_config.train_config data_config = pipeline_config.data_config feature_configs = config_util.get_compatible_feature_configs(pipeline_config) - if train_config.train_distribute != DistributionStrategy.NoStrategy\ - and train_config.sync_replicas: + if train_config.train_distribute != DistributionStrategy.NoStrategy and train_config.sync_replicas: logging.warning( 'will set sync_replicas to False, because train_distribute[%s] != NoStrategy' % pipeline_config.train_config.train_distribute) @@ -329,8 +339,8 @@ def _train_and_evaluate_impl(pipeline_config, train_steps = None if train_config.HasField('num_steps') and train_config.num_steps > 0: train_steps = train_config.num_steps - assert train_steps is not None or data_config.num_epochs > 0, ( - 'either num_steps and num_epochs must be set to an integer > 0.') + assert (train_steps is not None or data_config.num_epochs > 0 + ), 'either num_steps and num_epochs must be set to an integer > 0.' if train_steps and data_config.num_epochs: logging.info('Both num_steps and num_epochs are set.') @@ -352,21 +362,24 @@ def _train_and_evaluate_impl(pipeline_config, feature_configs, train_data, check_mode=check_mode, - **input_fn_kwargs) + **input_fn_kwargs, + ) # Currently only a single Eval Spec is allowed. train_spec = tf.estimator.TrainSpec( input_fn=train_input_fn, max_steps=train_steps) embedding_parallel = train_config.train_distribute in ( DistributionStrategy.SokStrategy, - DistributionStrategy.EmbeddingParallelStrategy) + DistributionStrategy.EmbeddingParallelStrategy, + ) if embedding_parallel: estimator.train( input_fn=train_input_fn, max_steps=train_spec.max_steps, hooks=list(train_spec.hooks), - saving_listeners=train_spec.saving_listeners) + saving_listeners=train_spec.saving_listeners, + ) train_input_fn.input_creator.stop() else: # create eval spec @@ -393,17 +406,20 @@ def _train_and_evaluate_impl(pipeline_config, input_fn=eval_input_fn, max_steps=fit_on_eval_steps, hooks=list(train_spec.hooks), - saving_listeners=train_spec.saving_listeners if hasattr( - train_spec, 'saving_listeners') else None) + saving_listeners=(train_spec.saving_listeners if hasattr( + train_spec, 'saving_listeners') else None), + ) logging.info('Finished training on eval data') # return estimator for custom training using estimator.train return estimator -def evaluate(pipeline_config, - eval_checkpoint_path='', - eval_data_path=None, - eval_result_filename='eval_result.txt'): +def evaluate( + pipeline_config, + eval_checkpoint_path='', + eval_data_path=None, + eval_result_filename='eval_result.txt', +): """Evaluate a EasyRec model defined in pipeline_config_path. Evaluate the model defined in pipeline_config_path on the eval data, @@ -440,6 +456,7 @@ def evaluate(pipeline_config, if 'TF_CONFIG' in os.environ: tf_config = estimator_utils.chief_to_master() from tensorflow.python.training import server_lib + if tf_config['task']['type'] == 'ps': cluster = tf.train.ClusterSpec(tf_config['cluster']) server = server_lib.Server( @@ -462,10 +479,12 @@ def evaluate(pipeline_config, input_iter = eval_spec.input_fn( mode=tf.estimator.ModeKeys.EVAL).make_one_shot_iterator() input_feas, input_lbls = input_iter.get_next() - from tensorflow.python.training.device_setter import replica_device_setter from tensorflow.python.framework.ops import device - from tensorflow.python.training.monitored_session import MonitoredSession - from tensorflow.python.training.monitored_session import ChiefSessionCreator + from tensorflow.python.training.device_setter import replica_device_setter + from tensorflow.python.training.monitored_session import ( # NOQA + ChiefSessionCreator, MonitoredSession, + ) + with device( replica_device_setter( worker_device='/job:master/task:0', cluster=cluster)): @@ -477,7 +496,8 @@ def evaluate(pipeline_config, chief_sess_creator = ChiefSessionCreator( master=server_target, checkpoint_filename_with_path=ckpt_path, - config=session_config) + config=session_config, + ) eval_metric_ops = estimator_spec.eval_metric_ops update_ops = [eval_metric_ops[x][1] for x in eval_metric_ops.keys()] metric_ops = {x: eval_metric_ops[x][0] for x in eval_metric_ops.keys()} @@ -521,10 +541,12 @@ def evaluate(pipeline_config, return eval_result -def distribute_evaluate(pipeline_config, - eval_checkpoint_path='', - eval_data_path=None, - eval_result_filename='distribute_eval_result.txt'): +def distribute_evaluate( + pipeline_config, + eval_checkpoint_path='', + eval_data_path=None, + eval_result_filename='distribute_eval_result.txt', +): """Evaluate a EasyRec model defined in pipeline_config_path. Evaluate the model defined in pipeline_config_path on the eval data, @@ -575,6 +597,7 @@ def distribute_evaluate(pipeline_config, tf_config = estimator_utils.chief_to_master() from tensorflow.python.training import server_lib + if tf_config['task']['type'] == 'ps': cluster = tf.train.ClusterSpec(tf_config['cluster']) server = server_lib.Server( @@ -600,12 +623,14 @@ def distribute_evaluate(pipeline_config, print('server_target = %s' % server_target) if server_target: - from tensorflow.python.training.device_setter import replica_device_setter from tensorflow.python.framework.ops import device - from tensorflow.python.training.monitored_session import MonitoredSession - from tensorflow.python.training.monitored_session import ChiefSessionCreator - from tensorflow.python.training.monitored_session import WorkerSessionCreator + from tensorflow.python.training.device_setter import replica_device_setter + from tensorflow.python.training.monitored_session import ( # NOQA + ChiefSessionCreator, MonitoredSession, WorkerSessionCreator, + ) + from easy_rec.python.utils.estimator_utils import EvaluateExitBarrierHook + cur_work_device = '/job:' + cur_job_name + '/task:' + str(cur_task_index) cur_ps_num = len(tf_config['cluster']['ps']) with device( @@ -627,7 +652,8 @@ def distribute_evaluate(pipeline_config, allow_soft_placement=True, log_device_placement=True, device_filters=['/job:ps', - '/job:worker/task:%d' % cur_task_index]) + '/job:worker/task:%d' % cur_task_index], + ) if cur_job_name == 'master': metric_variables = tf.get_collection(tf.GraphKeys.METRIC_VARIABLES) model_ready_for_local_init_op = tf.variables_initializer(metric_variables) @@ -642,7 +668,8 @@ def distribute_evaluate(pipeline_config, scaffold=cur_scaffold, master=server_target, checkpoint_filename_with_path=ckpt_path, - config=session_config) + config=session_config, + ) else: cur_sess_creator = WorkerSessionCreator( master=server_target, config=session_config) @@ -662,7 +689,8 @@ def distribute_evaluate(pipeline_config, with MonitoredSession( session_creator=cur_sess_creator, hooks=[cur_hooks], - stop_grace_period_secs=cur_stop_grace_period_sesc) as sess: + stop_grace_period_secs=cur_stop_grace_period_sesc, + ) as sess: while True: try: sess.run(update_op) @@ -731,12 +759,14 @@ def predict(pipeline_config, checkpoint_path='', data_path=None): return pred_result -def export(export_dir, - pipeline_config, - checkpoint_path='', - asset_files=None, - verbose=False, - **extra_params): +def export( + export_dir, + pipeline_config, + checkpoint_path='', + asset_files=None, + verbose=False, + **extra_params, +): """Export model defined in pipeline_config_path. Args: @@ -776,9 +806,9 @@ def export(export_dir, feature_configs = config_util.get_compatible_feature_configs(pipeline_config) # create estimator params = {'log_device_placement': verbose} + asset_file_dict = {} if asset_files: logging.info('will add asset files: %s' % asset_files) - asset_file_dict = {} for asset_file in asset_files.split(','): asset_file = asset_file.strip() if ':' not in asset_file or asset_file.startswith( @@ -807,23 +837,37 @@ def export(export_dir, if incr_save_type: extra_params['incr_update'][incr_save_type] = getattr( incr_save_config, incr_save_type) - return export_big_model_to_oss(export_dir, pipeline_config, extra_params, - serving_input_fn, estimator, ckpt_path, - verbose) + return export_big_model_to_oss( + export_dir, + pipeline_config, + extra_params, + serving_input_fn, + estimator, + ckpt_path, + verbose, + ) if 'redis_url' in extra_params: - return export_big_model(export_dir, pipeline_config, extra_params, - serving_input_fn, estimator, ckpt_path, verbose) + return export_big_model( + export_dir, + pipeline_config, + extra_params, + serving_input_fn, + estimator, + ckpt_path, + verbose, + ) final_export_dir = estimator.export_savedmodel( export_dir_base=export_dir, serving_input_receiver_fn=serving_input_fn, checkpoint_path=ckpt_path, - strip_default_attrs=True) + strip_default_attrs=True, + ) # add export ts as version info saved_model = saved_model_pb2.SavedModel() - if type(final_export_dir) not in [type(''), type(u'')]: + if type(final_export_dir) not in [type(''), type('')]: final_export_dir = final_export_dir.decode('utf-8') export_ts = [ x for x in final_export_dir.split('/') if x != '' and x is not None @@ -836,16 +880,34 @@ def export(export_dir, with tf.gfile.GFile(saved_pb_path, 'wb') as fout: fout.write(saved_model.SerializeToString()) + # modify export path + assets_dir = os.path.join(final_export_dir, 'assets') + for asset_rel_path in asset_file_dict: + if not os.path.dirname(asset_rel_path): + continue + asset_file = os.path.basename(asset_file_dict[asset_rel_path]) + asset_file = os.path.join(assets_dir, asset_file) + if not tf.gfile.Exists(asset_file): + logging.info('asset file %s does not exist' % asset_file) + continue + target_file = os.path.join(assets_dir, asset_rel_path) + target_dir = os.path.dirname(target_file) + if not tf.gfile.Exists(target_dir): + tf.gfile.MakeDirs(target_dir) + logging.info('will rename %s to %s' % (asset_file, target_file)) + tf.gfile.Rename(asset_file, target_file) logging.info('model has been exported to %s successfully' % final_export_dir) return final_export_dir -def export_checkpoint(pipeline_config=None, - export_path='', - checkpoint_path='', - asset_files=None, - verbose=False, - mode=tf.estimator.ModeKeys.PREDICT): +def export_checkpoint( + pipeline_config=None, + export_path='', + checkpoint_path='', + asset_files=None, + verbose=False, + mode=tf.estimator.ModeKeys.PREDICT, +): """Export the EasyRec model as checkpoint.""" pipeline_config = config_util.get_configs_from_pipeline_file(pipeline_config) if pipeline_config.fg_json_path: @@ -873,6 +935,7 @@ def export_checkpoint(pipeline_config=None, export_path=export_path, serving_input_receiver_fn=serving_input_fn, checkpoint_path=ckpt_path, - mode=mode) + mode=mode, + ) logging.info('model checkpoint has been exported successfully') diff --git a/easy_rec/python/model/autoint.py b/easy_rec/python/model/autoint.py index b7013486e..cf63ef4fd 100644 --- a/easy_rec/python/model/autoint.py +++ b/easy_rec/python/model/autoint.py @@ -23,8 +23,8 @@ def __init__(self, is_training=False): super(AutoInt, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'autoint', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof('model') == 'autoint', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._features, _ = self._input_layer(self._feature_dict, 'all') self._feature_num = len(self._model_config.feature_groups[0].feature_names) self._seq_key_num = 0 @@ -39,8 +39,10 @@ def __init__(self, fea_emb_dim_list = [] for feature_config in feature_configs: fea_emb_dim_list.append(feature_config.embedding_dim) - assert len(set(fea_emb_dim_list)) == 1 and len(fea_emb_dim_list) == self._feature_num, \ - 'AutoInt requires that all feature dimensions must be consistent.' + assert ( + len(set(fea_emb_dim_list)) == 1 and + len(fea_emb_dim_list) == self._feature_num + ), 'AutoInt requires that all feature dimensions must be consistent.' self._d_model = fea_emb_dim_list[0] self._head_num = self._model_config.multi_head_num @@ -51,7 +53,8 @@ def build_predict_graph(self): attention_fea = tf.reshape( self._features, - shape=[-1, self._feature_num + self._seq_key_num, self._d_model]) + shape=[-1, self._feature_num + self._seq_key_num, self._d_model], + ) for i in range(self._model_config.interacting_layer_num): attention_layer = multihead_attention.MultiHeadAttention( @@ -59,7 +62,8 @@ def build_predict_graph(self): head_size=self._head_size, l2_reg=self._l2_reg, use_res=True, - name='multi_head_self_attention_layer_%d' % i) + name='multi_head_self_attention_layer_%d' % i, + ) attention_fea = attention_layer(attention_fea) attention_fea = tf.reshape( diff --git a/easy_rec/python/model/cmbf.py b/easy_rec/python/model/cmbf.py index 0f0a8f3aa..6c8dd6b0d 100644 --- a/easy_rec/python/model/cmbf.py +++ b/easy_rec/python/model/cmbf.py @@ -28,12 +28,18 @@ def __init__(self, is_training=False): super(CMBF, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'cmbf', ( - 'invalid model config: %s' % self._model_config.WhichOneof('model')) - - self._cmbf_layer = cmbf.CMBF(model_config, feature_configs, features, - self._model_config.cmbf.config, - self._input_layer) + assert self._model_config.WhichOneof( + 'model' + ) == 'cmbf', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') + + self._cmbf_layer = cmbf.CMBF( + model_config, + feature_configs, + features, + self._model_config.cmbf.config, + self._input_layer, + ) self._model_config = self._model_config.cmbf def build_predict_graph(self): diff --git a/easy_rec/python/model/collaborative_metric_learning.py b/easy_rec/python/model/collaborative_metric_learning.py index b19537239..d2857d91c 100644 --- a/easy_rec/python/model/collaborative_metric_learning.py +++ b/easy_rec/python/model/collaborative_metric_learning.py @@ -1,7 +1,5 @@ import tensorflow as tf -from easy_rec.python.core.metrics import metric_learning_average_precision_at_k -from easy_rec.python.core.metrics import metric_learning_recall_at_k from easy_rec.python.layers import dnn from easy_rec.python.layers.common_layers import highway from easy_rec.python.loss.circle_loss import circle_loss @@ -13,6 +11,10 @@ from easy_rec.python.protos.collaborative_metric_learning_pb2 import CoMetricLearningI2I as MetricLearningI2IConfig # NOQA +from easy_rec.python.core.metrics import ( # NOQA + metric_learning_average_precision_at_k, metric_learning_recall_at_k, +) + if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -25,7 +27,8 @@ def __init__( feature_configs, # pipeline.feature_configs features, # same as model_fn input labels=None, - is_training=False): + is_training=False, + ): super(CoMetricLearningI2I, self).__init__(model_config, feature_configs, features, labels, is_training) model = self._model_config.WhichOneof('model') @@ -89,12 +92,14 @@ def build_predict_graph(self): self._highway_features[highway_cfg.input], training=self._is_training, trainable=True, - name='highway_%s_bn' % highway_cfg.input) + name='highway_%s_bn' % highway_cfg.input, + ) highway_fea = highway( highway_fea, highway_cfg.emb_size, activation=gelu, - scope='highway_%s' % _id) + scope='highway_%s' % _id, + ) print('highway_fea: ', highway_fea) self.input_features.append(highway_fea) @@ -108,7 +113,8 @@ def build_predict_graph(self): inputs=net_output, units=last_hidden, kernel_regularizer=self._l2_reg, - name='dnn/dnn_%d' % (num_dnn_layer - 1)) + name='dnn/dnn_%d' % (num_dnn_layer - 1), + ) if self._model_config.output_l2_normalized_emb: norm_emb = tf.nn.l2_normalize(tower_emb, axis=-1) @@ -135,7 +141,8 @@ def build_loss_graph(self): self.session_ids, self.loss.margin, self.loss.gamma, - embed_normed=emb_normed) + embed_normed=emb_normed, + ) elif self._loss_type == LossType.MULTI_SIMILARITY_LOSS: self._loss_dict['ms_loss'] = ms_loss( norm_emb, @@ -145,7 +152,8 @@ def build_loss_graph(self): self.loss.beta, self.loss.lamb, self.loss.eps, - embed_normed=emb_normed) + embed_normed=emb_normed, + ) else: raise ValueError('invalid loss type: %s' % LossType.Name(self._loss_type)) diff --git a/easy_rec/python/model/dat.py b/easy_rec/python/model/dat.py index 5c312299c..a622beee4 100644 --- a/easy_rec/python/model/dat.py +++ b/easy_rec/python/model/dat.py @@ -23,8 +23,10 @@ def __init__(self, is_training=False): super(DAT, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'dat', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof( + 'model' + ) == 'dat', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') feature_group_names = [ fg.group_name for fg in self._model_config.feature_groups @@ -63,7 +65,8 @@ def build_predict_graph(self): inputs=user_tower_emb, units=last_user_hidden, kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1)) + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), + ) num_item_dnn_layer = len(self.item_tower.dnn.hidden_units) last_item_hidden = self.item_tower.dnn.hidden_units.pop() @@ -77,7 +80,8 @@ def build_predict_graph(self): inputs=item_tower_emb, units=last_item_hidden, kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1)) + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), + ) user_tower_emb = self.norm(user_tower_emb) item_tower_emb = self.norm(item_tower_emb) @@ -126,9 +130,16 @@ def get_outputs(self): self._prediction_dict['probs'] = tf.nn.sigmoid( self._prediction_dict['logits']) return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb', 'augmented_p_u', 'augmented_p_i', 'augmented_a_u', - 'augmented_a_i' + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_tower_emb', + 'item_tower_emb', + 'augmented_p_u', + 'augmented_p_i', + 'augmented_a_u', + 'augmented_a_i', ] else: raise ValueError('invalid loss type: %s' % str(self._loss_type)) diff --git a/easy_rec/python/model/dbmtl.py b/easy_rec/python/model/dbmtl.py index 6c69d33ca..8424d125e 100644 --- a/easy_rec/python/model/dbmtl.py +++ b/easy_rec/python/model/dbmtl.py @@ -23,20 +23,27 @@ def __init__(self, is_training=False): super(DBMTL, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'dbmtl', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof('model') == 'dbmtl', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.dbmtl assert isinstance(self._model_config, DBMTLConfig) if self._model_config.HasField('bottom_cmbf'): - self._cmbf_layer = cmbf.CMBF(model_config, feature_configs, features, - self._model_config.bottom_cmbf, - self._input_layer) + self._cmbf_layer = cmbf.CMBF( + model_config, + feature_configs, + features, + self._model_config.bottom_cmbf, + self._input_layer, + ) elif self._model_config.HasField('bottom_uniter'): - self._uniter_layer = uniter.Uniter(model_config, feature_configs, - features, - self._model_config.bottom_uniter, - self._input_layer) + self._uniter_layer = uniter.Uniter( + model_config, + feature_configs, + features, + self._model_config.bottom_uniter, + self._input_layer, + ) elif not self.has_backbone: self._features, self._feature_list = self._input_layer( self._feature_dict, 'all') @@ -56,7 +63,8 @@ def build_predict_graph(self): self._model_config.bottom_dnn, self._l2_reg, name='bottom_dnn', - is_training=self._is_training) + is_training=self._is_training, + ) bottom_fea = bottom_dnn(self._features) else: bottom_fea = self._features @@ -67,7 +75,8 @@ def build_predict_graph(self): self._model_config.expert_dnn, l2_reg=self._l2_reg, num_task=self._task_num, - num_expert=self._model_config.num_expert) + num_expert=self._model_config.num_expert, + ) task_input_list = mmoe_layer(bottom_fea) else: task_input_list = [bottom_fea] * self._task_num @@ -81,7 +90,8 @@ def build_predict_graph(self): task_tower_cfg.dnn, self._l2_reg, name=tower_name + '/dnn', - is_training=self._is_training) + is_training=self._is_training, + ) tower_fea = tower_dnn(task_input_list[i]) tower_features[tower_name] = tower_fea else: @@ -96,7 +106,8 @@ def build_predict_graph(self): task_tower_cfg.relation_dnn, self._l2_reg, name=tower_name + '/relation_dnn', - is_training=self._is_training) + is_training=self._is_training, + ) tower_inputs = [tower_features[tower_name]] for relation_tower_name in task_tower_cfg.relation_tower_names: tower_inputs.append(relation_features[relation_tower_name]) @@ -109,7 +120,8 @@ def build_predict_graph(self): relation_fea, task_tower_cfg.num_class, kernel_regularizer=self._l2_reg, - name=tower_name + '/output') + name=tower_name + '/output', + ) tower_outputs[tower_name] = output_logits self._add_to_prediction_dict(tower_outputs) diff --git a/easy_rec/python/model/dcn.py b/easy_rec/python/model/dcn.py index fcfa7e780..aca03fe8e 100644 --- a/easy_rec/python/model/dcn.py +++ b/easy_rec/python/model/dcn.py @@ -22,8 +22,10 @@ def __init__(self, is_training=False): super(DCN, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'dcn', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof( + 'model' + ) == 'dcn', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.dcn assert isinstance(self._model_config, DCNConfig) diff --git a/easy_rec/python/model/deepfm.py b/easy_rec/python/model/deepfm.py index d1414c050..821174a0f 100644 --- a/easy_rec/python/model/deepfm.py +++ b/easy_rec/python/model/deepfm.py @@ -1,5 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. +import logging + import tensorflow as tf from easy_rec.python.layers import dnn @@ -21,8 +23,8 @@ def __init__(self, is_training=False): super(DeepFM, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'deepfm', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof('model') == 'deepfm', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.deepfm assert isinstance(self._model_config, DeepFMConfig) @@ -43,12 +45,22 @@ def build_input_layer(self, model_config, feature_configs): if not has_final: assert model_config.deepfm.wide_output_dim == model_config.num_class self._wide_output_dim = model_config.deepfm.wide_output_dim + if self._wide_output_dim != self._num_class: + logging.warning( + 'wide_output_dim not equal to 1, it is not a standard model') super(DeepFM, self).build_input_layer(model_config, feature_configs) def build_predict_graph(self): # Wide - wide_fea = tf.reduce_sum( - self._wide_features, axis=1, keepdims=True, name='wide_feature') + if self._num_class > 1 and self._wide_output_dim == self._num_class: + wide_shape = tf.shape(self._wide_features) + new_shape = tf.stack( + [-1, wide_shape[1] // self._num_class, self._num_class]) + wide_fea = tf.reshape(self._wide_features, new_shape) + wide_fea = tf.reduce_sum(wide_fea, axis=1, name='wide_feature') + else: + wide_fea = tf.reduce_sum( + self._wide_features, axis=1, keepdims=True, name='wide_feature') # FM fm_fea = fm.FM(name='fm_feature')(self._fm_features) @@ -62,8 +74,12 @@ def build_predict_graph(self): # Final if len(self._model_config.final_dnn.hidden_units) > 0: all_fea = tf.concat([wide_fea, fm_fea, deep_fea], axis=1) - final_dnn_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, - 'final_dnn', self._is_training) + final_dnn_layer = dnn.DNN( + self._model_config.final_dnn, + self._l2_reg, + 'final_dnn', + self._is_training, + ) all_fea = final_dnn_layer(all_fea) output = tf.layers.dense( all_fea, @@ -76,14 +92,16 @@ def build_predict_graph(self): fm_fea, self._num_class, kernel_regularizer=self._l2_reg, - name='fm_logits') + name='fm_logits', + ) else: fm_fea = tf.reduce_sum(fm_fea, 1, keepdims=True) deep_fea = tf.layers.dense( deep_fea, self._num_class, kernel_regularizer=self._l2_reg, - name='deep_logits') + name='deep_logits', + ) output = wide_fea + fm_fea + deep_fea self._add_to_prediction_dict(output) @@ -101,6 +119,6 @@ def build_feature_output_dict(self): tf.as_string(self._deep_features), axis=-1, separator=','), 'fm_outputs': tf.reduce_join( - tf.as_string(self._fm_outputs), axis=-1, separator=',') + tf.as_string(self._fm_outputs), axis=-1, separator=','), }) return outputs diff --git a/easy_rec/python/model/dlrm.py b/easy_rec/python/model/dlrm.py index 1b542ac58..a0d0f77ca 100755 --- a/easy_rec/python/model/dlrm.py +++ b/easy_rec/python/model/dlrm.py @@ -24,8 +24,9 @@ def __init__(self, is_training=False): super(DLRM, self).__init__(model_config, feature_configs, features, labels, is_training) - assert model_config.WhichOneof('model') == 'dlrm', \ - 'invalid model config: %s' % model_config.WhichOneof('model') + assert model_config.WhichOneof( + 'model' + ) == 'dlrm', 'invalid model config: %s' % model_config.WhichOneof('model') self._model_config = model_config.dlrm assert isinstance(self._model_config, DLRMConfig) assert self._input_layer.has_group( @@ -43,9 +44,11 @@ def build_predict_graph(self): if self._model_config.arch_interaction_op == 'cat': all_fea = tf.concat([dense_fea] + self._sparse_features, axis=1) elif self._model_config.arch_interaction_op == 'dot': - assert dense_fea.get_shape()[1] == self._sparse_features[0].get_shape()[1], \ - 'bot_dnn last hidden[%d] != sparse feature embedding_dim[%d]' % ( - dense_fea.get_shape()[1], self._sparse_features[0].get_shape()[1]) + assert dense_fea.get_shape()[1] == self._sparse_features[0].get_shape( + )[1], ('bot_dnn last hidden[%d] != sparse feature embedding_dim[%d]' % ( + dense_fea.get_shape()[1], + self._sparse_features[0].get_shape()[1], + )) all_feas = [dense_fea] + self._sparse_features all_feas = [x[:, None, :] for x in all_feas] diff --git a/easy_rec/python/model/dropoutnet.py b/easy_rec/python/model/dropoutnet.py index 683677531..e3e420c76 100644 --- a/easy_rec/python/model/dropoutnet.py +++ b/easy_rec/python/model/dropoutnet.py @@ -8,9 +8,10 @@ from easy_rec.python.protos.loss_pb2 import LossType from easy_rec.python.utils.proto_util import copy_obj +from easy_rec.python.loss.softmax_loss_with_negative_mining import ( # NOQA + softmax_loss_with_negative_mining,) from easy_rec.python.protos.dropoutnet_pb2 import DropoutNet as DropoutNetConfig # NOQA -from easy_rec.python.loss.softmax_loss_with_negative_mining import softmax_loss_with_negative_mining # NOQA -from easy_rec.python.protos.dropoutnet_pb2 import DropoutNet as DropoutNetConfig # NOQA + if tf.__version__ >= '2.0': tf = tf.compat.v1 losses = tf.losses @@ -42,10 +43,8 @@ def __init__(self, super(DropoutNet, self).__init__(model_config, feature_configs, features, labels, is_training) self._losses = self._model_config.losses - assert self._model_config.WhichOneof( - 'model' - ) == 'dropoutnet', 'invalid model config: %s' % self._model_config.WhichOneof( - 'model') + assert self._model_config.WhichOneof('model') == 'dropoutnet', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.dropoutnet assert isinstance(self._model_config, DropoutNetConfig) @@ -86,16 +85,26 @@ def build_predict_graph(self): with tf.name_scope('user_tower'): user_features = [] if self.user_content_feature is not None: - user_content_dnn = dnn.DNN(self.user_content_layers, self._l2_reg, - 'user_content', self._is_training) + user_content_dnn = dnn.DNN( + self.user_content_layers, + self._l2_reg, + 'user_content', + self._is_training, + ) content_feature = user_content_dnn(self.user_content_feature) user_features.append(content_feature) if self.user_preference_feature is not None: user_prefer_feature = bernoulli_dropout( - self.user_preference_feature, self._model_config.user_dropout_rate, - self._is_training) - user_prefer_dnn = dnn.DNN(self.user_preference_layers, self._l2_reg, - 'user_preference', self._is_training) + self.user_preference_feature, + self._model_config.user_dropout_rate, + self._is_training, + ) + user_prefer_dnn = dnn.DNN( + self.user_preference_layers, + self._l2_reg, + 'user_preference', + self._is_training, + ) prefer_feature = user_prefer_dnn(user_prefer_feature) user_features.append(prefer_feature) @@ -108,22 +117,33 @@ def build_predict_graph(self): inputs=user_hidden, units=last_user_hidden, kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1)) + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), + ) # --------------------------build item tower----------------------------------- with tf.name_scope('item_tower'): item_features = [] if self.item_content_feature is not None: - item_content_dnn = dnn.DNN(self.item_content_layers, self._l2_reg, - 'item_content', self._is_training) + item_content_dnn = dnn.DNN( + self.item_content_layers, + self._l2_reg, + 'item_content', + self._is_training, + ) content_feature = item_content_dnn(self.item_content_feature) item_features.append(content_feature) if self.item_preference_feature is not None: item_prefer_feature = bernoulli_dropout( - self.item_preference_feature, self._model_config.item_dropout_rate, - self._is_training) - item_prefer_dnn = dnn.DNN(self.item_preference_layers, self._l2_reg, - 'item_preference', self._is_training) + self.item_preference_feature, + self._model_config.item_dropout_rate, + self._is_training, + ) + item_prefer_dnn = dnn.DNN( + self.item_preference_layers, + self._l2_reg, + 'item_preference', + self._is_training, + ) prefer_feature = item_prefer_dnn(item_prefer_feature) item_features.append(prefer_feature) @@ -136,7 +156,8 @@ def build_predict_graph(self): inputs=item_hidden, units=last_item_hidden, kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1)) + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), + ) user_emb = tf.nn.l2_normalize(user_tower_emb, axis=-1) item_emb = tf.nn.l2_normalize(item_tower_emb, axis=-1) @@ -168,7 +189,8 @@ def build_loss_graph(self): weights=self._sample_weight, margin=self._model_config.softmax_loss.margin, gamma=self._model_config.softmax_loss.gamma, - t=self._model_config.softmax_loss.coefficient_of_support_vector) + t=self._model_config.softmax_loss.coefficient_of_support_vector, + ) self._loss_dict['softmax_loss'] = loss_value * loss.weight elif loss.loss_type == LossType.PAIR_WISE_LOSS: loss_value = pairwise_loss(labels, logits) @@ -181,6 +203,7 @@ def build_loss_graph(self): def build_metric_graph(self, eval_config): from easy_rec.python.core.easyrec_metrics import metrics_tf as metrics + metric_dict = {} labels = list(self._labels.values())[0] sim_score = self._prediction_dict['similarity'] diff --git a/easy_rec/python/model/dssm.py b/easy_rec/python/model/dssm.py index e35d69030..740e02257 100644 --- a/easy_rec/python/model/dssm.py +++ b/easy_rec/python/model/dssm.py @@ -24,8 +24,10 @@ def __init__(self, is_training=False): super(DSSM, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'dssm', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof( + 'model' + ) == 'dssm', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.dssm assert isinstance(self._model_config, DSSMConfig) @@ -48,7 +50,8 @@ def build_predict_graph(self): inputs=user_tower_emb, units=last_user_hidden, kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1)) + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), + ) num_item_dnn_layer = len(self.item_tower.dnn.hidden_units) last_item_hidden = self.item_tower.dnn.hidden_units.pop() @@ -59,7 +62,8 @@ def build_predict_graph(self): inputs=item_tower_emb, units=last_item_hidden, kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1)) + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), + ) if self._model_config.simi_func == Similarity.COSINE: user_tower_emb = self.norm(user_tower_emb) @@ -108,8 +112,12 @@ def build_predict_graph(self): def get_outputs(self): if self._loss_type == LossType.CLASSIFICATION: return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb' + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_tower_emb', + 'item_tower_emb', ] elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: self._prediction_dict['logits'] = tf.squeeze( @@ -117,8 +125,12 @@ def get_outputs(self): self._prediction_dict['probs'] = tf.nn.sigmoid( self._prediction_dict['logits']) return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_tower_emb', - 'item_tower_emb' + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_tower_emb', + 'item_tower_emb', ] elif self._loss_type == LossType.L2_LOSS: return ['y', 'user_emb', 'item_emb', 'user_tower_emb', 'item_tower_emb'] diff --git a/easy_rec/python/model/dssm_senet.py b/easy_rec/python/model/dssm_senet.py index c84d52161..4676e72eb 100644 --- a/easy_rec/python/model/dssm_senet.py +++ b/easy_rec/python/model/dssm_senet.py @@ -25,26 +25,33 @@ def __init__(self, features, labels=None, is_training=False): - MatchModel.__init__(self, model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'dssm_senet', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof('model') == 'dssm_senet', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.dssm_senet assert isinstance(self._model_config, DSSM_SENet_Config) # copy_obj so that any modification will not affect original config self.user_tower = copy_obj(self._model_config.user_tower) - self.user_seq_features, self.user_plain_features, self.user_feature_list = self._input_layer( + ( + self.user_seq_features, + self.user_plain_features, + self.user_feature_list, + ) = self._input_layer( self._feature_dict, 'user', is_combine=False) self.user_num_fields = len(self.user_feature_list) # copy_obj so that any modification will not affect original config self.item_tower = copy_obj(self._model_config.item_tower) - self.item_seq_features, self.item_plain_features, self.item_feature_list = self._input_layer( + ( + self.item_seq_features, + self.item_plain_features, + self.item_feature_list, + ) = self._input_layer( self._feature_dict, 'item', is_combine=False) self.item_num_fields = len(self.item_feature_list) @@ -57,7 +64,8 @@ def build_predict_graph(self): num_squeeze_group=self.user_tower.senet.num_squeeze_group, reduction_ratio=self.user_tower.senet.reduction_ratio, l2_reg=self._l2_reg, - name='user_senet') + name='user_senet', + ) user_senet_output_list = user_senet(self.user_feature_list) user_senet_output = tf.concat(user_senet_output_list, axis=-1) @@ -70,14 +78,16 @@ def build_predict_graph(self): inputs=user_tower_emb, units=last_user_hidden, kernel_regularizer=self._l2_reg, - name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1)) + name='user_dnn/dnn_%d' % (num_user_dnn_layer - 1), + ) item_senet = senet.SENet( num_fields=self.item_num_fields, num_squeeze_group=self.item_tower.senet.num_squeeze_group, reduction_ratio=self.item_tower.senet.reduction_ratio, l2_reg=self._l2_reg, - name='item_senet') + name='item_senet', + ) item_senet_output_list = item_senet(self.item_feature_list) item_senet_output = tf.concat(item_senet_output_list, axis=-1) @@ -91,7 +101,8 @@ def build_predict_graph(self): inputs=item_tower_emb, units=last_item_hidden, kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1)) + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), + ) if self._model_config.simi_func == Similarity.COSINE: user_tower_emb = self.norm(user_tower_emb) diff --git a/easy_rec/python/model/easy_rec_estimator.py b/easy_rec/python/model/easy_rec_estimator.py index 95385936c..074f43623 100644 --- a/easy_rec/python/model/easy_rec_estimator.py +++ b/easy_rec/python/model/easy_rec_estimator.py @@ -22,25 +22,22 @@ from easy_rec.python.builders import optimizer_builder from easy_rec.python.compat import optimizers from easy_rec.python.compat import sync_replicas_optimizer -from easy_rec.python.compat.early_stopping import custom_early_stop_hook -from easy_rec.python.compat.early_stopping import deadline_stop_hook -from easy_rec.python.compat.early_stopping import find_early_stop_var -from easy_rec.python.compat.early_stopping import oss_stop_hook -from easy_rec.python.compat.early_stopping import stop_if_no_decrease_hook -from easy_rec.python.compat.early_stopping import stop_if_no_increase_hook from easy_rec.python.compat.ops import GraphKeys from easy_rec.python.input.input import Input from easy_rec.python.layers.utils import _tensor_to_tensorinfo from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig from easy_rec.python.protos.train_pb2 import DistributionStrategy -from easy_rec.python.utils import constant -from easy_rec.python.utils import embedding_utils -from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils import hvd_utils -from easy_rec.python.utils import pai_util from easy_rec.python.utils.multi_optimizer import MultiOptimizer -from easy_rec.python.compat.embedding_parallel_saver import EmbeddingParallelSaver # NOQA +from easy_rec.python.compat.early_stopping import ( # NOQA + custom_early_stop_hook, deadline_stop_hook, find_early_stop_var, + oss_stop_hook, stop_if_no_decrease_hook, stop_if_no_increase_hook, +) +from easy_rec.python.compat.embedding_parallel_saver import ( # NOQA + EmbeddingParallelSaver,) +from easy_rec.python.utils import ( # NOQA + constant, embedding_utils, estimator_utils, hvd_utils, pai_util, +) try: import horovod.tensorflow as hvd @@ -49,6 +46,7 @@ try: from sparse_operation_kit import experiment as sok + from easy_rec.python.compat import sok_optimizer except Exception: sok = None @@ -70,7 +68,8 @@ def __init__(self, pipeline_config, model_cls, run_config, params): model_fn=self._model_fn, model_dir=pipeline_config.model_dir, config=run_config, - params=params) + params=params, + ) def evaluate(self, input_fn, @@ -141,7 +140,8 @@ def export_config(self): def embedding_parallel(self): return self.train_config.train_distribute in ( DistributionStrategy.SokStrategy, - DistributionStrategy.EmbeddingParallelStrategy) + DistributionStrategy.EmbeddingParallelStrategy, + ) @property def saver_cls(self): @@ -192,7 +192,8 @@ def _train_model_fn(self, features, labels, run_config): dtype=tf.string, shape=[task_num], collections=[tf.GraphKeys.GLOBAL_VARIABLES, Input.DATA_OFFSET], - trainable=False) + trainable=False, + ) update_offset = tf.assign(data_offset_var[task_index], features[Input.DATA_OFFSET]) ops.add_to_collection(tf.GraphKeys.UPDATE_OPS, update_offset) @@ -226,9 +227,11 @@ def _train_model_fn(self, features, labels, run_config): tf.summary.scalar('learning_rate', learning_rate[0]) all_opts.append(opt) grouped_vars = model.get_grouped_vars(len(all_opts)) - assert len(grouped_vars) == len(optimizer_config), \ - 'the number of var group(%d) != the number of optimizers(%d)' \ - % (len(grouped_vars), len(optimizer_config)) + assert len(grouped_vars) == len(optimizer_config), ( + 'the number of var group(%d) != the number of optimizers(%d)' % ( + len(grouped_vars), + len(optimizer_config), + )) optimizer = MultiOptimizer(all_opts, grouped_vars) if self.train_config.train_distribute == DistributionStrategy.SokStrategy: @@ -236,8 +239,7 @@ def _train_model_fn(self, features, labels, run_config): hooks = [] if estimator_utils.has_hvd(): - assert not self.train_config.sync_replicas, \ - 'sync_replicas should not be set when using horovod' + assert not self.train_config.sync_replicas, 'sync_replicas should not be set when using horovod' bcast_hook = hvd_utils.BroadcastGlobalVariablesHook(0) hooks.append(bcast_hook) @@ -250,18 +252,19 @@ def _train_model_fn(self, features, labels, run_config): optimizer, replicas_to_aggregate=run_config.num_worker_replicas, total_num_replicas=run_config.num_worker_replicas, - sparse_accumulator_type=self.train_config.sparse_accumulator_type) + sparse_accumulator_type=self.train_config.sparse_accumulator_type, + ) else: optimizer = sync_replicas_optimizer.SyncReplicasOptimizer( optimizer, replicas_to_aggregate=run_config.num_worker_replicas, - total_num_replicas=run_config.num_worker_replicas) + total_num_replicas=run_config.num_worker_replicas, + ) hooks.append( optimizer.make_session_run_hook(run_config.is_chief, num_tokens=0)) # add barrier for no strategy case - if run_config.num_worker_replicas > 1 and \ - self.train_config.train_distribute == DistributionStrategy.NoStrategy: + if run_config.num_worker_replicas > 1 and self.train_config.train_distribute == DistributionStrategy.NoStrategy: hooks.append( estimator_utils.ExitBarrierHook(run_config.num_worker_replicas, run_config.is_chief, self.model_dir)) @@ -275,21 +278,24 @@ def _train_model_fn(self, features, labels, run_config): self, eval_dir=eval_dir, custom_stop_func=self.export_config.early_stop_func, - custom_stop_func_params=self.export_config.early_stop_params)) + custom_stop_func_params=self.export_config.early_stop_params, + )) elif self.export_config.metric_bigger: hooks.append( stop_if_no_increase_hook( self, self.export_config.best_exporter_metric, self.export_config.max_check_steps, - eval_dir=eval_dir)) + eval_dir=eval_dir, + )) else: hooks.append( stop_if_no_decrease_hook( self, self.export_config.best_exporter_metric, self.export_config.max_check_steps, - eval_dir=eval_dir)) + eval_dir=eval_dir, + )) if self.train_config.enable_oss_stop_signal: hooks.append(oss_stop_hook(self)) @@ -350,7 +356,8 @@ def _train_model_fn(self, features, labels, run_config): self._pipeline_config.data_config.chief_redundant, name='', # Preventing scope prefix on all variables. incr_save=(self.incr_save_config is not None), - embedding_parallel=self.embedding_parallel) + embedding_parallel=self.embedding_parallel, + ) # online evaluation metric_update_op_dict = None @@ -376,7 +383,8 @@ def _train_model_fn(self, features, labels, run_config): fine_tune_ckpt, include_global_step=False, ckpt_var_map_path=fine_tune_ckpt_var_map, - force_restore_shape_compatible=force_restore) + force_restore_shape_compatible=force_restore, + ) if restore_hook is not None: hooks.append(restore_hook) @@ -392,21 +400,22 @@ def _train_model_fn(self, features, labels, run_config): logging_hook = basic_session_run_hooks.LoggingTensorHook( logging_dict, every_n_iter=log_step_count_steps, - formatter=estimator_utils.tensor_log_format_func) + formatter=estimator_utils.tensor_log_format_func, + ) hooks.append(logging_hook) if self.train_config.train_distribute in [ DistributionStrategy.CollectiveAllReduceStrategy, DistributionStrategy.MirroredStrategy, - DistributionStrategy.MultiWorkerMirroredStrategy + DistributionStrategy.MultiWorkerMirroredStrategy, ]: # for multi worker strategy, we could not replace the # inner CheckpointSaverHook, so just use it. scaffold = tf.train.Scaffold() else: - var_list = ( - tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) + - tf.get_collection(tf.GraphKeys.SAVEABLE_OBJECTS)) + var_list = tf.get_collection( + tf.GraphKeys.GLOBAL_VARIABLES) + tf.get_collection( + tf.GraphKeys.SAVEABLE_OBJECTS) # exclude data_offset_var var_list = [x for x in var_list if x != data_offset_var] @@ -437,10 +446,12 @@ def _train_model_fn(self, features, labels, run_config): var_list=var_list, sharded=True, max_to_keep=self.train_config.keep_checkpoint_max, - save_relative_paths=True), + save_relative_paths=True, + ), local_init_op=tf.group(local_init_ops), ready_for_local_init_op=tf.report_uninitialized_variables( - var_list=initialize_var_list)) + var_list=initialize_var_list), + ) # saver hook saver_hook = estimator_utils.CheckpointSaverHook( checkpoint_dir=self.model_dir, @@ -449,7 +460,8 @@ def _train_model_fn(self, features, labels, run_config): scaffold=scaffold, write_graph=self.train_config.write_graph, data_offset_var=data_offset_var, - increment_save_config=self.incr_save_config) + increment_save_config=self.incr_save_config, + ) if estimator_utils.is_chief() or self.embedding_parallel: hooks.append(saver_hook) if estimator_utils.is_chief(): @@ -469,7 +481,8 @@ def _train_model_fn(self, features, labels, run_config): predictions=predict_dict, train_op=train_op, scaffold=scaffold, - training_hooks=hooks) + training_hooks=hooks, + ) def _eval_model_fn(self, features, labels, run_config): tf.keras.backend.set_learning_phase(0) @@ -492,16 +505,17 @@ def _eval_model_fn(self, features, labels, run_config): metric_dict['loss/loss/' + loss_key] = tf.metrics.mean(loss_tensor) tf.logging.info('metric_dict keys: %s' % metric_dict.keys()) - var_list = ( - ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + - ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS)) + var_list = ops.get_collection( + ops.GraphKeys.GLOBAL_VARIABLES) + ops.get_collection( + ops.GraphKeys.SAVEABLE_OBJECTS) metric_variables = ops.get_collection(ops.GraphKeys.METRIC_VARIABLES) model_ready_for_local_init_op = tf.variables_initializer(metric_variables) scaffold = tf.train.Scaffold( saver=self.saver_cls( var_list=var_list, sharded=True, save_relative_paths=True), - ready_for_local_init_op=model_ready_for_local_init_op) + ready_for_local_init_op=model_ready_for_local_init_op, + ) end = time.time() tf.logging.info('eval graph construct finished. Time %.3fs' % (end - start)) return tf.estimator.EstimatorSpec( @@ -509,7 +523,8 @@ def _eval_model_fn(self, features, labels, run_config): loss=loss, scaffold=scaffold, predictions=predict_dict, - eval_metric_ops=metric_dict) + eval_metric_ops=metric_dict, + ) def _distribute_eval_model_fn(self, features, labels, run_config): tf.keras.backend.set_learning_phase(0) @@ -562,7 +577,8 @@ def _distribute_eval_model_fn(self, features, labels, run_config): loss=loss, predictions=predict_dict, eval_metric_ops=metric_dict, - scaffold=scaffold) + scaffold=scaffold, + ) def _export_model_fn(self, features, labels, run_config, params): tf.keras.backend.set_learning_phase(0) @@ -571,7 +587,8 @@ def _export_model_fn(self, features, labels, run_config, params): self.feature_configs, features, labels=None, - is_training=False) + is_training=False, + ) model.build_predict_graph() export_config = self._pipeline_config.export_config @@ -599,7 +616,8 @@ def _export_model_fn(self, features, labels, run_config, params): if gfile.Exists(pipeline_path): ops.add_to_collection( tf.GraphKeys.ASSET_FILEPATHS, - tf.constant(pipeline_path, dtype=tf.string, name='pipeline.config')) + tf.constant(pipeline_path, dtype=tf.string, name='pipeline.config'), + ) else: print('train pipeline_path(%s) does not exist' % pipeline_path) @@ -624,16 +642,22 @@ def _export_model_fn(self, features, labels, run_config, params): for asset_file in export_config.asset_files: if asset_file.startswith('!'): asset_file = asset_file[1:] - _, asset_name = os.path.split(asset_file) + if ':' not in asset_file or asset_file.startswith( + 'oss:') or asset_file.startswith('hdfs:'): + _, asset_name = os.path.split(asset_file) + else: + asset_name, asset_file = asset_file.split(':', 1) ops.add_to_collection( ops.GraphKeys.ASSET_FILEPATHS, - tf.constant(asset_file, dtype=tf.string, name=asset_name)) + tf.constant(asset_file, dtype=tf.string, name=asset_name), + ) elif 'asset_files' in params: for asset_name in params['asset_files']: asset_file = params['asset_files'][asset_name] ops.add_to_collection( tf.GraphKeys.ASSET_FILEPATHS, - tf.constant(asset_file, dtype=tf.string, name=asset_name)) + tf.constant(asset_file, dtype=tf.string, name=asset_name), + ) if self._pipeline_config.HasField('fg_json_path'): fg_path = self._pipeline_config.fg_json_path @@ -641,11 +665,12 @@ def _export_model_fn(self, features, labels, run_config, params): fg_path = fg_path[1:] ops.add_to_collection( tf.GraphKeys.ASSET_FILEPATHS, - tf.constant(fg_path, dtype=tf.string, name='fg.json')) + tf.constant(fg_path, dtype=tf.string, name='fg.json'), + ) - var_list = ( - ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + - ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS)) + var_list = ops.get_collection( + ops.GraphKeys.GLOBAL_VARIABLES) + ops.get_collection( + ops.GraphKeys.SAVEABLE_OBJECTS) scaffold = tf.train.Scaffold( saver=self.saver_cls( @@ -656,7 +681,8 @@ def _export_model_fn(self, features, labels, run_config, params): loss=None, scaffold=scaffold, predictions=outputs, - export_outputs=export_outputs) + export_outputs=export_outputs, + ) def _model_fn(self, features, labels, mode, config, params): os.environ['tf.estimator.mode'] = mode @@ -714,11 +740,13 @@ def _write_rtp_inputs_to_col(features): else: col[0] = json.dumps(feature_info_map) - def export_checkpoint(self, - export_path=None, - serving_input_receiver_fn=None, - checkpoint_path=None, - mode=tf.estimator.ModeKeys.PREDICT): + def export_checkpoint( + self, + export_path=None, + serving_input_receiver_fn=None, + checkpoint_path=None, + mode=tf.estimator.ModeKeys.PREDICT, + ): with context.graph_mode(): if not checkpoint_path: # Locate the latest checkpoint @@ -731,7 +759,8 @@ def export_checkpoint(self, features=input_receiver.features, labels=getattr(input_receiver, 'labels', None), mode=mode, - config=self.config) + config=self.config, + ) with tf_session.Session(config=self._session_config) as session: graph_saver = estimator_spec.scaffold.saver or saver.Saver( sharded=True) diff --git a/easy_rec/python/model/easy_rec_model.py b/easy_rec/python/model/easy_rec_model.py index f2408ba47..8087f5e68 100644 --- a/easy_rec/python/model/easy_rec_model.py +++ b/easy_rec/python/model/easy_rec_model.py @@ -23,15 +23,17 @@ try: import horovod.tensorflow as hvd - from sparse_operation_kit.experiment import raw_ops as dynamic_variable_ops from sparse_operation_kit import experiment as sok + from sparse_operation_kit.experiment import raw_ops as dynamic_variable_ops except Exception: dynamic_variable_ops = None sok = None try: from tensorflow.python.framework.load_library import load_op_library + import easy_rec + load_embed_lib_path = os.path.join(easy_rec.ops_dir, 'libload_embed.so') load_embed_lib = load_op_library(load_embed_lib_path) except Exception as ex: @@ -105,7 +107,8 @@ def build_backbone_network(self): self._base_model_config.backbone, self._feature_dict, input_layer=self._input_layer, - l2_reg=self._l2_reg) + l2_reg=self._l2_reg, + ) return None @property @@ -122,7 +125,7 @@ def backbone(self): 'metric_dict': self._metric_dict, 'prediction_dict': self._prediction_dict, 'labels': self._labels, - constant.SAMPLE_WEIGHT: self._sample_weight + constant.SAMPLE_WEIGHT: self._sample_weight, } return self._backbone_net(self._is_training, **kwargs) return None @@ -144,8 +147,8 @@ def l2_regularization(self): model_config = getattr(self._base_model_config, self._base_model_config.WhichOneof('model')) l2_regularization = 0.0 - if hasattr(model_config, 'dense_regularization') and \ - model_config.HasField('dense_regularization'): + if hasattr(model_config, 'dense_regularization') and model_config.HasField( + 'dense_regularization'): # backward compatibility logging.warn( 'dense_regularization is deprecated, please use l2_regularization') @@ -162,10 +165,12 @@ def build_input_layer(self, model_config, feature_configs): ev_params=self._global_ev_params, embedding_regularizer=self._emb_reg, kernel_regularizer=self._l2_reg, - variational_dropout_config=model_config.variational_dropout - if model_config.HasField('variational_dropout') else None, + variational_dropout_config=(model_config.variational_dropout if + model_config.HasField('variational_dropout') + else None), is_training=self._is_training, - is_predicting=self._is_predicting) + is_predicting=self._is_predicting, + ) @abstractmethod def build_predict_graph(self): @@ -216,11 +221,13 @@ def build_rtp_output_dict(self): """For exporting: get output nodes for RTP infering.""" return {} - def restore(self, - ckpt_path, - include_global_step=False, - ckpt_var_map_path='', - force_restore_shape_compatible=False): + def restore( + self, + ckpt_path, + include_global_step=False, + ckpt_var_map_path='', + force_restore_shape_compatible=False, + ): """Restore variables from ckpt_path. steps: @@ -266,12 +273,14 @@ def restore(self, var_shape, variable[0].dtype, variable, - partitions=[len(variable)] + [1] * (len(var_shape) - 1)) + partitions=[len(variable)] + [1] * (len(var_shape) - 1), + ) else: var_shape = variable.shape.as_list() if ckpt_var_shape == var_shape: - vars_in_ckpt[variable_name] = list(variable) if isinstance( - variable, variables.PartitionedVariable) else variable + vars_in_ckpt[variable_name] = ( + list(variable) if isinstance( + variable, variables.PartitionedVariable) else variable) elif len(ckpt_var_shape) == len(var_shape): if force_restore_shape_compatible: # create a variable compatible with checkpoint to restore @@ -285,7 +294,8 @@ def restore(self, # add to a special collection for easy reference # by tf.get_collection('T_E_M_P_RESTROE') collections=['T_E_M_P_RESTROE'], - dtype=dtype) + dtype=dtype, + ) vars_in_ckpt[variable_name] = tmp_var incompatible_shape_var_map[variable] = tmp_var print('incompatible restore %s[%s, %s]' % @@ -293,16 +303,21 @@ def restore(self, else: logging.warning( 'Variable [%s] is available in checkpoint, but ' - 'incompatible shape with model variable.', variable_name) + 'incompatible shape with model variable.', + variable_name, + ) else: logging.warning( 'Variable [%s] is available in checkpoint, but ' - 'incompatible shape dims with model variable.', variable_name) + 'incompatible shape dims with model variable.', + variable_name, + ) elif 'EmbeddingVariable' in str(type(variable)): if '%s-keys' % variable_name not in ckpt_var2shape_map: continue print('restore embedding_variable %s' % variable_name) from tensorflow.python.training import saver + names_to_saveables = saver.BaseSaverBuilder.OpListToDict([variable]) saveable_objects = [] for name, op in names_to_saveables.items(): @@ -316,6 +331,7 @@ def restore(self, continue print('restore partitioned embedding_variable %s' % variable_name) from tensorflow.python.training import saver + for part_var in variable: names_to_saveables = saver.BaseSaverBuilder.OpListToDict([part_var]) saveable_objects = [] @@ -331,7 +347,8 @@ def restore(self, task_num=hvd.size(), embed_dim=variable._dimension, var_name='embed-' + variable.name.replace('/', '__'), - ckpt_path=ckpt_path) + ckpt_path=ckpt_path, + ) with ops.control_dependencies([variable._initializer_op]): variable._initializer_op = dynamic_variable_ops.dummy_var_assign( variable.handle, keys, vals) @@ -440,8 +457,10 @@ def get_restore_filter(self): for x in self._base_model_config.restore_filters ] - return restore_filter.CombineFilter(all_filters, - restore_filter.Logical.AND), None + return ( + restore_filter.CombineFilter(all_filters, restore_filter.Logical.AND), + None, + ) def get_grouped_vars(self, opt_num): """Group the vars into different optimization groups. diff --git a/easy_rec/python/model/esmm.py b/easy_rec/python/model/esmm.py index 50567ae63..f998f9734 100644 --- a/easy_rec/python/model/esmm.py +++ b/easy_rec/python/model/esmm.py @@ -24,8 +24,10 @@ def __init__(self, is_training=False): super(ESMM, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'esmm', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof( + 'model' + ) == 'esmm', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.esmm assert isinstance(self._model_config, ESMMConfig) @@ -50,8 +52,7 @@ def __init__(self, self._ctr_tower_cfg = self._model_config.ctr_tower self._init_towers([self._cvr_tower_cfg, self._ctr_tower_cfg]) - assert self._model_config.ctr_tower.loss_type == LossType.CLASSIFICATION, \ - 'ctr tower must be binary classification.' + assert self._model_config.ctr_tower.loss_type == LossType.CLASSIFICATION, 'ctr tower must be binary classification.' for task_tower_cfg in self._task_towers: assert task_tower_cfg.num_class == 1, 'Does not support multiclass classification problem' @@ -84,7 +85,8 @@ def build_loss_graph(self): cvr_loss = tf.losses.mean_squared_error( labels=ctcvr_label, predictions=self._prediction_dict['y_ctcvr'], - weights=self._sample_weight) + weights=self._sample_weight, + ) self._loss_dict['weighted_l2_loss_%s' % cvr_tower_name] = self._cvr_tower_cfg.weight * cvr_loss _labels = tf.cast(self._labels[ctr_label_name], tf.float32) @@ -123,7 +125,8 @@ def build_metric_graph(self, eval_config): loss_type=self._cvr_tower_cfg.loss_type, label_name=ctcvr_label_name, num_class=self._cvr_tower_cfg.num_class, - suffix='_ctcvr')) + suffix='_ctcvr', + )) # CVR metric cvr_label_masked_name = cvr_label_name + '_masked' @@ -140,7 +143,8 @@ def build_metric_graph(self, eval_config): loss_type=self._cvr_tower_cfg.loss_type, label_name=cvr_label_masked_name, num_class=self._cvr_tower_cfg.num_class, - suffix='_%s_masked' % cvr_tower_name)) + suffix='_%s_masked' % cvr_tower_name, + )) for metric in self._ctr_tower_cfg.metrics_set: # CTR metric @@ -150,7 +154,8 @@ def build_metric_graph(self, eval_config): loss_type=self._ctr_tower_cfg.loss_type, label_name=ctr_label_name, num_class=self._ctr_tower_cfg.num_class, - suffix='_%s' % ctr_tower_name)) + suffix='_%s' % ctr_tower_name, + )) return metric_dict def _add_to_prediction_dict(self, output): @@ -158,14 +163,16 @@ def _add_to_prediction_dict(self, output): if self._cvr_tower_cfg.loss_type == LossType.CLASSIFICATION: prob = tf.multiply( self._prediction_dict['probs_%s' % self._cvr_tower_cfg.tower_name], - self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name]) + self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name], + ) # pctcvr = pctr * pcvr self._prediction_dict['probs_ctcvr'] = prob else: prob = tf.multiply( self._prediction_dict['y_%s' % self._cvr_tower_cfg.tower_name], - self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name]) + self._prediction_dict['probs_%s' % self._ctr_tower_cfg.tower_name], + ) # pctcvr = pctr * pcvr self._prediction_dict['y_ctcvr'] = prob @@ -197,30 +204,34 @@ def build_predict_graph(self): self._cvr_tower_cfg.dnn, self._l2_reg, name=cvr_tower_name, - is_training=self._is_training) + is_training=self._is_training, + ) cvr_tower_output = dnn_model(all_fea) cvr_tower_output = tf.layers.dense( inputs=cvr_tower_output, units=1, kernel_regularizer=self._l2_reg, - name='%s/dnn_output' % cvr_tower_name) + name='%s/dnn_output' % cvr_tower_name, + ) ctr_tower_name = self._ctr_tower_cfg.tower_name dnn_model = dnn.DNN( self._ctr_tower_cfg.dnn, self._l2_reg, name=ctr_tower_name, - is_training=self._is_training) + is_training=self._is_training, + ) ctr_tower_output = dnn_model(all_fea) ctr_tower_output = tf.layers.dense( inputs=ctr_tower_output, units=1, kernel_regularizer=self._l2_reg, - name='%s/dnn_output' % ctr_tower_name) + name='%s/dnn_output' % ctr_tower_name, + ) tower_outputs = { cvr_tower_name: cvr_tower_output, - ctr_tower_name: ctr_tower_output + ctr_tower_name: ctr_tower_output, } self._add_to_prediction_dict(tower_outputs) return self._prediction_dict diff --git a/easy_rec/python/model/fm.py b/easy_rec/python/model/fm.py index 357628d3e..c54042684 100644 --- a/easy_rec/python/model/fm.py +++ b/easy_rec/python/model/fm.py @@ -22,8 +22,10 @@ def __init__(self, is_training=False): super(FM, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'fm', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof( + 'model' + ) == 'fm', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.fm assert isinstance(self._model_config, FMConfig) @@ -47,14 +49,17 @@ def build_predict_graph(self): fm_fea, self._num_class, kernel_regularizer=self._l2_reg, - name='fm_logits') + name='fm_logits', + ) else: fm_fea = tf.reduce_sum(fm_fea, 1, keepdims=True) bias = tf.get_variable( - 'fm_bias', [self._num_class], + 'fm_bias', + [self._num_class], initializer=tf.zeros_initializer(), - trainable=True) + trainable=True, + ) output = wide_fea + fm_fea output = tf.nn.bias_add(output, bias) diff --git a/easy_rec/python/model/mind.py b/easy_rec/python/model/mind.py index 0e47f79d8..747f829c4 100644 --- a/easy_rec/python/model/mind.py +++ b/easy_rec/python/model/mind.py @@ -28,8 +28,10 @@ def __init__(self, is_training=False): super(MIND, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'mind', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof( + 'model' + ) == 'mind', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.mind self._hist_seq_features, _, _ = self._input_layer( @@ -79,17 +81,21 @@ def build_predict_graph(self): # sum pooling over the features hist_embed_dims = [x.get_shape()[-1] for x in hist_seq_feas] for i in range(1, len(hist_embed_dims)): - assert hist_embed_dims[i] == hist_embed_dims[0], \ - 'all hist seq must have the same embedding shape, but: %s' \ - % str(hist_embed_dims) + assert hist_embed_dims[i] == hist_embed_dims[0], ( + 'all hist seq must have the same embedding shape, but: %s' % + str(hist_embed_dims)) hist_seq_feas = tf.add_n(hist_seq_feas) / len(hist_seq_feas) else: hist_seq_feas = tf.concat(hist_seq_feas, axis=2) - if self._model_config.HasField('pre_capsule_dnn') and \ - len(self._model_config.pre_capsule_dnn.hidden_units) > 0: - pre_dnn_layer = dnn.DNN(self._model_config.pre_capsule_dnn, self._l2_reg, - 'pre_capsule_dnn', self._is_training) + if self._model_config.HasField('pre_capsule_dnn') and len( + self._model_config.pre_capsule_dnn.hidden_units) > 0: + pre_dnn_layer = dnn.DNN( + self._model_config.pre_capsule_dnn, + self._l2_reg, + 'pre_capsule_dnn', + self._is_training, + ) hist_seq_feas = pre_dnn_layer(hist_seq_feas) if time_id_fea is not None: @@ -122,7 +128,8 @@ def build_predict_graph(self): self._user_features, training=self._is_training, trainable=True, - name='user_fea_bn') + name='user_fea_bn', + ) user_dnn = dnn.DNN(self.user_dnn, self._l2_reg, 'user_dnn', self._is_training) user_features = user_dnn(user_features) @@ -144,7 +151,8 @@ def build_predict_graph(self): inputs=user_interests, units=last_hidden, kernel_regularizer=self._l2_reg, - name='concat_dnn/dnn_%d' % (num_concat_dnn_layer - 1)) + name='concat_dnn/dnn_%d' % (num_concat_dnn_layer - 1), + ) num_item_dnn_layer = len(self.item_dnn.hidden_units) last_item_hidden = self.item_dnn.hidden_units.pop() @@ -155,10 +163,12 @@ def build_predict_graph(self): inputs=item_tower_emb, units=last_item_hidden, kernel_regularizer=self._l2_reg, - name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1)) + name='item_dnn/dnn_%d' % (num_item_dnn_layer - 1), + ) assert self._model_config.simi_func in [ - Similarity.COSINE, Similarity.INNER_PRODUCT + Similarity.COSINE, + Similarity.INNER_PRODUCT, ] if self._model_config.simi_func == Similarity.COSINE: @@ -236,7 +246,8 @@ def build_predict_graph(self): self._prediction_dict['user_emb'] = tf.reduce_join( tf.reduce_join(tf.as_string(user_interests), axis=-1, separator=','), axis=-1, - separator='|') + separator='|', + ) self._prediction_dict['user_emb_num'] = num_high_capsules self._prediction_dict['item_emb'] = tf.reduce_join( tf.as_string(item_tower_emb), axis=-1, separator=',') @@ -300,11 +311,12 @@ def _build_interest_simi(self): def build_metric_graph(self, eval_config): from easy_rec.python.core.easyrec_metrics import metrics_tf as metrics + # build interest metric interest_simi, capsule_simi = self._build_interest_simi() metric_dict = { 'interest_similarity': metrics.mean(interest_simi), - 'capsule_similarity': metrics.mean(capsule_simi) + 'capsule_similarity': metrics.mean(capsule_simi), } if self._is_point_wise: metric_dict.update(self._build_point_wise_metric_graph(eval_config)) @@ -314,7 +326,8 @@ def build_metric_graph(self, eval_config): for metric in eval_config.metrics_set: if metric.WhichOneof('metric') == 'recall_at_topk': assert self._loss_type in [ - LossType.CLASSIFICATION, LossType.SOFTMAX_CROSS_ENTROPY + LossType.CLASSIFICATION, + LossType.SOFTMAX_CROSS_ENTROPY, ] if metric.recall_at_topk.topk not in recall_at_topks: recall_at_topks.append(metric.recall_at_topk.topk) @@ -360,12 +373,14 @@ def build_metric_graph(self, eval_config): labels=simple_lbls, predictions=simple_item_sim, k=topk, - name='interests_recall_at_%d' % topk) + name='interests_recall_at_%d' % topk, + ) metric_dict['interests_neg_sam_recall@%d' % topk] = metrics.recall_at_k( labels=simple_lbls_v2, predictions=simple_item_sim_v2, k=topk, - name='interests_recall_neg_sam_at_%d' % topk) + name='interests_recall_neg_sam_at_%d' % topk, + ) logits = self._prediction_dict['logits'] pos_item_logits = tf.gather_nd(logits[:batch_size, :batch_size], @@ -379,24 +394,32 @@ def build_metric_graph(self, eval_config): labels=simple_lbls, predictions=logits, k=topk, - name='recall_at_%d' % topk) + name='recall_at_%d' % topk, + ) metric_dict['recall_neg_sam@%d' % topk] = metrics.recall_at_k( labels=labels_v2, predictions=logits_v2, k=topk, - name='recall_neg_sam_at_%d' % topk) + name='recall_neg_sam_at_%d' % topk, + ) eval_logits = logits[:, :batch_size] eval_logits = tf.cond( - batch_size < topk, lambda: tf.pad( - eval_logits, [[0, 0], [0, topk - batch_size]], + batch_size < topk, + lambda: tf.pad( + eval_logits, + [[0, 0], [0, topk - batch_size]], mode='CONSTANT', constant_values=-1e32, - name='pad_eval_logits'), lambda: eval_logits) + name='pad_eval_logits', + ), + lambda: eval_logits, + ) metric_dict['recall_in_batch@%d' % topk] = metrics.recall_at_k( labels=simple_lbls, predictions=eval_logits, k=topk, - name='recall_in_batch_at_%d' % topk) + name='recall_in_batch_at_%d' % topk, + ) # batch_size num_interest if hard_neg_indices is not None: @@ -409,7 +432,8 @@ def build_metric_graph(self, eval_config): hard_neg_mask = tf.scatter_nd( hard_neg_indices, tf.ones_like(hard_neg_sim, dtype=tf.float32), - shape=hard_neg_shape) + shape=hard_neg_shape, + ) hard_neg_sim = tf.scatter_nd(hard_neg_indices, hard_neg_sim, hard_neg_shape) hard_neg_sim = hard_neg_sim - (1 - hard_neg_mask) * 1e32 @@ -424,8 +448,13 @@ def build_metric_graph(self, eval_config): def get_outputs(self): if self._loss_type == LossType.CLASSIFICATION: return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_emb_num', - 'user_interests', 'item_tower_emb' + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_emb_num', + 'user_interests', + 'item_tower_emb', ] elif self._loss_type == LossType.SOFTMAX_CROSS_ENTROPY: self._prediction_dict['logits'] = tf.squeeze( @@ -433,13 +462,22 @@ def get_outputs(self): self._prediction_dict['probs'] = tf.nn.sigmoid( self._prediction_dict['logits']) return [ - 'logits', 'probs', 'user_emb', 'item_emb', 'user_emb_num', - 'user_interests', 'item_tower_emb' + 'logits', + 'probs', + 'user_emb', + 'item_emb', + 'user_emb_num', + 'user_interests', + 'item_tower_emb', ] elif self._loss_type == LossType.L2_LOSS: return [ - 'y', 'user_emb', 'item_emb', 'user_emb_num', 'user_interests', - 'item_tower_emb' + 'y', + 'user_emb', + 'item_emb', + 'user_emb_num', + 'user_interests', + 'item_tower_emb', ] else: raise ValueError('invalid loss type: %s' % str(self._loss_type)) diff --git a/easy_rec/python/model/mmoe.py b/easy_rec/python/model/mmoe.py index 3cc644f6d..23a8a6643 100644 --- a/easy_rec/python/model/mmoe.py +++ b/easy_rec/python/model/mmoe.py @@ -21,8 +21,10 @@ def __init__(self, is_training=False): super(MMoE, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'mmoe', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof( + 'model' + ) == 'mmoe', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.mmoe assert isinstance(self._model_config, MMoEConfig) @@ -38,12 +40,15 @@ def build_predict_graph(self): self._model_config.expert_dnn, l2_reg=self._l2_reg, num_task=self._task_num, - num_expert=self._model_config.num_expert) + num_expert=self._model_config.num_expert, + ) else: # For backward compatibility with original mmoe layer config - mmoe_layer = mmoe.MMOE([x.dnn for x in self._model_config.experts], - l2_reg=self._l2_reg, - num_task=self._task_num) + mmoe_layer = mmoe.MMOE( + [x.dnn for x in self._model_config.experts], + l2_reg=self._l2_reg, + num_task=self._task_num, + ) task_input_list = mmoe_layer(self._features) tower_outputs = {} @@ -55,7 +60,8 @@ def build_predict_graph(self): task_tower_cfg.dnn, self._l2_reg, name=tower_name, - is_training=self._is_training) + is_training=self._is_training, + ) tower_output = tower_dnn(task_input_list[i]) else: tower_output = task_input_list[i] @@ -63,7 +69,8 @@ def build_predict_graph(self): inputs=tower_output, units=task_tower_cfg.num_class, kernel_regularizer=self._l2_reg, - name='dnn_output_%d' % i) + name='dnn_output_%d' % i, + ) tower_outputs[tower_name] = tower_output self._add_to_prediction_dict(tower_outputs) diff --git a/easy_rec/python/model/multi_task_model.py b/easy_rec/python/model/multi_task_model.py index aa102104c..9fa67e10b 100644 --- a/easy_rec/python/model/multi_task_model.py +++ b/easy_rec/python/model/multi_task_model.py @@ -61,7 +61,8 @@ def build_predict_graph(self): task_tower_cfg.dnn, self._l2_reg, name=tower_name, - is_training=self._is_training) + is_training=self._is_training, + ) tower_output = tower_dnn(task_input_list[i]) else: tower_output = task_input_list[i] @@ -78,7 +79,8 @@ def build_predict_graph(self): task_tower_cfg.relation_dnn, self._l2_reg, name=tower_name + '/relation_dnn', - is_training=self._is_training) + is_training=self._is_training, + ) tower_inputs = [tower_features[tower_name]] for relation_tower_name in task_tower_cfg.relation_tower_names: tower_inputs.append(relation_features[relation_tower_name]) @@ -93,7 +95,8 @@ def build_predict_graph(self): relation_fea, task_tower_cfg.num_class, kernel_regularizer=self._l2_reg, - name=tower_name + '/output') + name=tower_name + '/output', + ) tower_outputs[tower_name] = output_logits self._add_to_prediction_dict(tower_outputs) @@ -104,9 +107,9 @@ def _init_towers(self, task_tower_configs): self._task_towers = task_tower_configs self._task_num = len(task_tower_configs) for i, task_tower_config in enumerate(task_tower_configs): - assert isinstance(task_tower_config, tower_pb2.TaskTower) or \ - isinstance(task_tower_config, tower_pb2.BayesTaskTower), \ - 'task_tower_config must be a instance of tower_pb2.TaskTower or tower_pb2.BayesTaskTower' + assert isinstance(task_tower_config, tower_pb2.TaskTower) or isinstance( + task_tower_config, tower_pb2.BayesTaskTower + ), 'task_tower_config must be a instance of tower_pb2.TaskTower or tower_pb2.BayesTaskTower' tower_name = task_tower_config.tower_name # For label backward compatibility with list @@ -130,7 +133,8 @@ def _add_to_prediction_dict(self, output): output[tower_name], loss_type=task_tower_cfg.loss_type, num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name)) + suffix='_%s' % tower_name, + )) else: for loss in task_tower_cfg.losses: self._prediction_dict.update( @@ -138,7 +142,8 @@ def _add_to_prediction_dict(self, output): output[tower_name], loss_type=loss.loss_type, num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name)) + suffix='_%s' % tower_name, + )) def build_metric_graph(self, eval_config): """Build metric graph for multi task model.""" @@ -154,7 +159,8 @@ def build_metric_graph(self, eval_config): loss_type=loss_types, label_name=self._label_name_dict[tower_name], num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name)) + suffix='_%s' % tower_name, + )) return self._metric_dict def build_loss_weight(self): @@ -207,20 +213,22 @@ def build_loss_graph(self): if task_tower_cfg.use_sample_weight: loss_weight *= self._sample_weight - if hasattr(task_tower_cfg, 'task_space_indicator_label') and \ - task_tower_cfg.HasField('task_space_indicator_label'): + if hasattr(task_tower_cfg, 'task_space_indicator_label' + ) and task_tower_cfg.HasField('task_space_indicator_label'): in_task_space = tf.to_float( self._labels[task_tower_cfg.task_space_indicator_label] > 0) loss_weight = loss_weight * ( task_tower_cfg.in_task_space_weight * in_task_space + task_tower_cfg.out_task_space_weight * (1 - in_task_space)) - if task_tower_cfg.HasField('task_space_indicator_name') and \ - task_tower_cfg.HasField('task_space_indicator_value'): + if task_tower_cfg.HasField( + 'task_space_indicator_name') and task_tower_cfg.HasField( + 'task_space_indicator_value'): in_task_space = tf.to_float( tf.equal( self._feature_dict[task_tower_cfg.task_space_indicator_name], - task_tower_cfg.task_space_indicator_value)) + task_tower_cfg.task_space_indicator_value, + )) loss_weight = loss_weight * ( task_tower_cfg.in_task_space_weight * in_task_space + task_tower_cfg.out_task_space_weight * (1 - in_task_space)) @@ -234,7 +242,8 @@ def build_loss_graph(self): label_name=self._label_name_dict[tower_name], loss_weight=loss_weight, num_class=task_tower_cfg.num_class, - suffix='_%s' % tower_name) + suffix='_%s' % tower_name, + ) for loss_name in loss_dict.keys(): loss_dict[loss_name] = loss_dict[loss_name] * task_loss_weight[0] else: @@ -259,7 +268,8 @@ def build_loss_graph(self): num_class=task_tower_cfg.num_class, suffix='_%s' % tower_name, loss_name=loss.loss_name, - loss_param=loss_param) + loss_param=loss_param, + ) for i, loss_name in enumerate(loss_ops): loss_value = loss_ops[loss_name] if loss.learn_loss_weight: @@ -290,7 +300,8 @@ def get_outputs(self): self._get_outputs_impl( task_tower_cfg.loss_type, task_tower_cfg.num_class, - suffix='_%s' % tower_name)) + suffix='_%s' % tower_name, + )) else: for loss in task_tower_cfg.losses: if loss.loss_type == LossType.ORDER_CALIBRATE_LOSS: @@ -299,5 +310,6 @@ def get_outputs(self): self._get_outputs_impl( loss.loss_type, task_tower_cfg.num_class, - suffix='_%s' % tower_name)) + suffix='_%s' % tower_name, + )) return list(set(outputs)) diff --git a/easy_rec/python/model/multi_tower.py b/easy_rec/python/model/multi_tower.py index 5cdd89ba5..0663602e8 100644 --- a/easy_rec/python/model/multi_tower.py +++ b/easy_rec/python/model/multi_tower.py @@ -44,7 +44,8 @@ def build_predict_graph(self): tower_fea, training=self._is_training, trainable=True, - name='%s_fea_bn' % tower_name) + name='%s_fea_bn' % tower_name, + ) tower_dnn_layer = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training) diff --git a/easy_rec/python/model/multi_tower_bst.py b/easy_rec/python/model/multi_tower_bst.py index 4cbc9fd29..17bea71a6 100644 --- a/easy_rec/python/model/multi_tower_bst.py +++ b/easy_rec/python/model/multi_tower_bst.py @@ -31,9 +31,10 @@ def __init__(self, feature_configs, model_config.seq_att_groups, embedding_regularizer=self._emb_reg, - ev_params=self._global_ev_params) - assert self._model_config.WhichOneof('model') == 'multi_tower', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + ev_params=self._global_ev_params, + ) + assert self._model_config.WhichOneof('model') == 'multi_tower', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.multi_tower assert isinstance(self._model_config, MultiTowerConfig) @@ -69,7 +70,8 @@ def dnn_net(self, net, dnn_units, name): net, units=units, activation=tf.nn.relu, - name='%s_%d' % (name, idx)) + name='%s_%d' % (name, idx), + ) else: net = tf.layers.dense( net, units=units, activation=None, name='%s_%d' % (name, idx)) @@ -89,7 +91,7 @@ def attention_net(self, net, dim, cur_seq_len, seq_size, name): mask = tf.concat([hist_mask, cur_id_mask], axis=1) # [B, seq_size] masks = tf.reshape(tf.tile(mask, [1, seq_size]), (-1, seq_size, seq_size)) # [B, seq_size, seq_size] - padding = tf.ones_like(scores) * (-2**32 + 1) + padding = tf.ones_like(scores) * (-(2**32) + 1) scores = tf.where(masks, scores, padding) # [B, seq_size, seq_size] # Scale @@ -110,7 +112,8 @@ def multi_head_att_net(self, id_cols, head_count, emb_dim, seq_len, seq_size): part_cols_emd_dim, seq_len, seq_size, - name='multi_head_%d' % start_idx) + name='multi_head_%d' % start_idx, + ) multi_head_attention_res.append(part_attention_net) multi_head_attention_res_net = tf.concat(multi_head_attention_res, axis=2) multi_head_attention_res_net = self.dnn_net( @@ -125,16 +128,23 @@ def add_and_norm(self, net_1, net_2, emb_dim, name): return net def bst(self, bst_fea, seq_size, head_count, name): - cur_id, hist_id_col, seq_len = bst_fea['key'], bst_fea[ - 'hist_seq_emb'], bst_fea['hist_seq_len'] + cur_id, hist_id_col, seq_len = ( + bst_fea['key'], + bst_fea['hist_seq_emb'], + bst_fea['hist_seq_len'], + ) cur_batch_max_seq_len = tf.shape(hist_id_col)[1] hist_id_col = tf.cond( - tf.constant(seq_size) > cur_batch_max_seq_len, lambda: tf.pad( - hist_id_col, [[0, 0], [0, seq_size - cur_batch_max_seq_len - 1], - [0, 0]], 'CONSTANT'), - lambda: tf.slice(hist_id_col, [0, 0, 0], [-1, seq_size - 1, -1])) + tf.constant(seq_size) > cur_batch_max_seq_len, + lambda: tf.pad( + hist_id_col, + [[0, 0], [0, seq_size - cur_batch_max_seq_len - 1], [0, 0]], + 'CONSTANT', + ), + lambda: tf.slice(hist_id_col, [0, 0, 0], [-1, seq_size - 1, -1]), + ) all_ids = tf.concat([hist_id_col, tf.expand_dims(cur_id, 1)], axis=1) # b, seq_size, emb_dim @@ -160,7 +170,8 @@ def build_predict_graph(self): tower_fea, training=self._is_training, trainable=True, - name='%s_fea_bn' % tower_name) + name='%s_fea_bn' % tower_name, + ) tower_dnn = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training) tower_fea = tower_dnn(tower_fea) @@ -176,7 +187,8 @@ def build_predict_graph(self): tower_fea, seq_size=tower_seq_len, head_count=tower_multi_head_size, - name=tower_name) + name=tower_name, + ) tower_fea_arr.append(tower_fea) all_fea = tf.concat(tower_fea_arr, axis=1) diff --git a/easy_rec/python/model/multi_tower_din.py b/easy_rec/python/model/multi_tower_din.py index e586da1cf..04f34c753 100644 --- a/easy_rec/python/model/multi_tower_din.py +++ b/easy_rec/python/model/multi_tower_din.py @@ -29,9 +29,10 @@ def __init__(self, feature_configs, model_config.seq_att_groups, embedding_regularizer=self._emb_reg, - ev_params=self._global_ev_params) - assert self._model_config.WhichOneof('model') == 'multi_tower', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + ev_params=self._global_ev_params, + ) + assert self._model_config.WhichOneof('model') == 'multi_tower', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.multi_tower assert isinstance(self._model_config, MultiTowerConfig) @@ -60,8 +61,11 @@ def __init__(self, self._din_tower_features.append(tower_feature) def din(self, dnn_config, deep_fea, name): - cur_id, hist_id_col, seq_len = deep_fea['key'], deep_fea[ - 'hist_seq_emb'], deep_fea['hist_seq_len'] + cur_id, hist_id_col, seq_len = ( + deep_fea['key'], + deep_fea['hist_seq_emb'], + deep_fea['hist_seq_len'], + ) seq_max_len = tf.shape(hist_id_col)[1] emb_dim = hist_id_col.shape[2] @@ -72,7 +76,8 @@ def din(self, dnn_config, deep_fea, name): din_net = tf.concat( [cur_ids, hist_id_col, cur_ids - hist_id_col, cur_ids * hist_id_col], - axis=-1) # (B, seq_max_len, emb_dim*4) + axis=-1, + ) # (B, seq_max_len, emb_dim*4) din_layer = dnn.DNN( dnn_config, @@ -80,13 +85,14 @@ def din(self, dnn_config, deep_fea, name): name, self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True) + last_layer_no_batch_norm=True, + ) din_net = din_layer(din_net) scores = tf.reshape(din_net, [-1, 1, seq_max_len]) # (B, 1, ?) seq_len = tf.expand_dims(seq_len, 1) mask = tf.sequence_mask(seq_len) - padding = tf.ones_like(scores) * (-2**32 + 1) + padding = tf.ones_like(scores) * (-(2**32) + 1) scores = tf.where(mask, scores, padding) # [B, 1, seq_max_len] # Scale @@ -106,7 +112,8 @@ def build_predict_graph(self): tower_fea, training=self._is_training, trainable=True, - name='%s_fea_bn' % tower_name) + name='%s_fea_bn' % tower_name, + ) dnn_layer = dnn.DNN(tower.dnn, self._l2_reg, '%s_dnn' % tower_name, self._is_training) tower_fea = dnn_layer(tower_fea) diff --git a/easy_rec/python/model/multi_tower_recall.py b/easy_rec/python/model/multi_tower_recall.py index 8f576944e..bf42a2cb8 100644 --- a/easy_rec/python/model/multi_tower_recall.py +++ b/easy_rec/python/model/multi_tower_recall.py @@ -31,24 +31,34 @@ def __init__(self, self.item_tower_feature, _ = self._input_layer(self._feature_dict, 'item') def build_predict_graph(self): - user_tower_feature = self.user_tower_feature batch_size = tf.shape(user_tower_feature)[0] pos_item_feature = self.item_tower_feature[:batch_size] neg_item_feature = self.item_tower_feature[batch_size:] - item_tower_feature = tf.concat([ - pos_item_feature[:, tf.newaxis, :], - tf.tile( - neg_item_feature[tf.newaxis, :, :], multiples=[batch_size, 1, 1]) - ], - axis=1) # noqa: E126 - - user_dnn = dnn.DNN(self._model_config.user_tower.dnn, self._l2_reg, - 'user_dnn', self._is_training) + item_tower_feature = tf.concat( + [ + pos_item_feature[:, tf.newaxis, :], + tf.tile( + neg_item_feature[tf.newaxis, :, :], + multiples=[batch_size, 1, 1]), + ], + axis=1, + ) # noqa: E126 + + user_dnn = dnn.DNN( + self._model_config.user_tower.dnn, + self._l2_reg, + 'user_dnn', + self._is_training, + ) user_tower_emb = user_dnn(user_tower_feature) - item_dnn = dnn.DNN(self._model_config.item_tower.dnn, self._l2_reg, - 'item_dnn', self._is_training) + item_dnn = dnn.DNN( + self._model_config.item_tower.dnn, + self._l2_reg, + 'item_dnn', + self._is_training, + ) item_tower_emb = item_dnn(item_tower_feature) item_tower_emb = tf.reshape(item_tower_emb, tf.shape(user_tower_emb)) diff --git a/easy_rec/python/model/pdn.py b/easy_rec/python/model/pdn.py index 7325beb1c..195bbba1f 100644 --- a/easy_rec/python/model/pdn.py +++ b/easy_rec/python/model/pdn.py @@ -23,8 +23,10 @@ def __init__(self, is_training=False): super(PDN, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'pdn', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof( + 'model' + ) == 'pdn', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.pdn self._user_features, _ = self._input_layer(self._feature_dict, 'user') @@ -88,7 +90,8 @@ def _build_trigger_net(self): 'trigger_dnn', self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True) + last_layer_no_batch_norm=True, + ) # output: N x seq_len x d, d is usually set to 1 trigger_out = trigger_dnn_layer(trigger_merge_fea) @@ -99,9 +102,11 @@ def _build_trigger_net(self): tf.reduce_join( tf.as_string(trigger_out, precision=4, shortest=True), axis=2, - separator=','), + separator=',', + ), axis=1, - separator=';') + separator=';', + ) return trigger_out def _build_similarity_net(self): @@ -126,7 +131,8 @@ def _build_similarity_net(self): 'sim_dnn', self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True) + last_layer_no_batch_norm=True, + ) # output: N x seq_len x 1 sim_out = sim_dnn_layer(sim_seq_concat) # exp(x): map (-inf, inf) to (0, inf) @@ -138,25 +144,28 @@ def _build_similarity_net(self): axis=2, separator=','), axis=1, - separator=';') + separator=';', + ) return sim_out def _build_direct_net(self): - if self._model_config.HasField('direct_user_dnn') and \ - self._model_config.HasField('direct_item_dnn'): + if self._model_config.HasField( + 'direct_user_dnn') and self._model_config.HasField('direct_item_dnn'): direct_user_layer = dnn.DNN( self._model_config.direct_user_dnn, 'direct_user_dnn', self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True) + last_layer_no_batch_norm=True, + ) direct_user_out = direct_user_layer(self._user_features) direct_item_layer = dnn.DNN( self._model_config.direct_item_dnn, 'direct_item_dnn', self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True) + last_layer_no_batch_norm=True, + ) direct_item_out = direct_item_layer(self._item_features) if self._model_config.simi_func == Similarity.COSINE: @@ -172,12 +181,14 @@ def _build_direct_net(self): 'direct_net/sim_w', dtype=tf.float32, shape=(1), - initializer=tf.ones_initializer()) + initializer=tf.ones_initializer(), + ) sim_b = tf.get_variable( 'direct_net/sim_b', dtype=tf.float32, shape=(1), - initializer=tf.zeros_initializer()) + initializer=tf.zeros_initializer(), + ) direct_logits = direct_logits * tf.abs(sim_w) + sim_b return tf.nn.softplus(direct_logits) @@ -193,7 +204,8 @@ def _build_bias_net(self): 'bias_dnn', self._is_training, last_layer_no_activation=True, - last_layer_no_batch_norm=True) + last_layer_no_batch_norm=True, + ) bias_logits = bias_dnn_layer(self._bias_features) return tf.nn.softplus(bias_logits) else: diff --git a/easy_rec/python/model/ple.py b/easy_rec/python/model/ple.py index e04781bcd..6159c8272 100644 --- a/easy_rec/python/model/ple.py +++ b/easy_rec/python/model/ple.py @@ -20,8 +20,10 @@ def __init__(self, is_training=False): super(PLE, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'ple', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof( + 'model' + ) == 'ple', 'invalid model config: %s' % self._model_config.WhichOneof( + 'model') self._model_config = self._model_config.ple assert isinstance(self._model_config, PLEConfig) @@ -40,7 +42,8 @@ def gate(self, selector_fea, vec_feas, name): units=len(vec_feas), kernel_regularizer=self._l2_reg, activation=None, - name=name + '_gate/dnn') + name=name + '_gate/dnn', + ) gate = tf.nn.softmax(gate, axis=1) gate = tf.expand_dims(gate, -1) task_input = tf.multiply(vec, gate) @@ -54,17 +57,26 @@ def experts_layer(self, deep_fea, expert_num, experts_cfg, name): experts_cfg, self._l2_reg, name=name + '_expert_%d/dnn' % expert_id, - is_training=self._is_training) + is_training=self._is_training, + ) tower_output = tower_dnn(deep_fea) tower_outputs.append(tower_output) return tower_outputs - def CGC_layer(self, extraction_networks_cfg, extraction_network_fea, - shared_expert_fea, final_flag): + def CGC_layer( + self, + extraction_networks_cfg, + extraction_network_fea, + shared_expert_fea, + final_flag, + ): layer_name = extraction_networks_cfg.network_name expert_shared_out = self.experts_layer( - shared_expert_fea, extraction_networks_cfg.share_num, - extraction_networks_cfg.share_expert_net, layer_name + '_share/dnn') + shared_expert_fea, + extraction_networks_cfg.share_num, + extraction_networks_cfg.share_expert_net, + layer_name + '_share/dnn', + ) experts_outs = [] cgc_layer_outs = [] @@ -73,7 +85,9 @@ def CGC_layer(self, extraction_networks_cfg, extraction_network_fea, experts_out = self.experts_layer( extraction_network_fea[task_idx], extraction_networks_cfg.expert_num_per_task, - extraction_networks_cfg.task_expert_net, name) + extraction_networks_cfg.task_expert_net, + name, + ) cgc_layer_out = self.gate(extraction_network_fea[task_idx], experts_out + expert_shared_out, name) experts_outs.extend(experts_out) @@ -82,9 +96,11 @@ def CGC_layer(self, extraction_networks_cfg, extraction_network_fea, if final_flag: shared_layer_out = None else: - shared_layer_out = self.gate(shared_expert_fea, - experts_outs + expert_shared_out, - layer_name + '_share') + shared_layer_out = self.gate( + shared_expert_fea, + experts_outs + expert_shared_out, + layer_name + '_share', + ) return cgc_layer_outs, shared_layer_out def build_predict_graph(self): @@ -96,8 +112,11 @@ def build_predict_graph(self): if idx == len(self._model_config.extraction_networks) - 1: final_flag = True extraction_network_fea, shared_expert_fea = self.CGC_layer( - extraction_network, extraction_network_fea, shared_expert_fea, - final_flag) + extraction_network, + extraction_network_fea, + shared_expert_fea, + final_flag, + ) tower_outputs = {} for i, task_tower_cfg in enumerate(self._model_config.task_towers): @@ -106,14 +125,16 @@ def build_predict_graph(self): task_tower_cfg.dnn, self._l2_reg, name=tower_name, - is_training=self._is_training) + is_training=self._is_training, + ) tower_output = tower_dnn(extraction_network_fea[i]) tower_output = tf.layers.dense( inputs=tower_output, units=task_tower_cfg.num_class, kernel_regularizer=self._l2_reg, - name='dnn_output_%d' % i) + name='dnn_output_%d' % i, + ) tower_outputs[tower_name] = tower_output self._add_to_prediction_dict(tower_outputs) diff --git a/easy_rec/python/model/rank_model.py b/easy_rec/python/model/rank_model.py index dc3771daf..e08a80e3c 100644 --- a/easy_rec/python/model/rank_model.py +++ b/easy_rec/python/model/rank_model.py @@ -9,7 +9,8 @@ from easy_rec.python.model.easy_rec_model import EasyRecModel from easy_rec.python.protos.loss_pb2 import LossType -from easy_rec.python.loss.zero_inflated_lognormal import zero_inflated_lognormal_pred # NOQA +from easy_rec.python.loss.zero_inflated_lognormal import ( # NOQA + zero_inflated_lognormal_pred,) if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -57,14 +58,19 @@ def _output_to_prediction_impl(self, output, loss_type, num_class=1, - suffix=''): + suffix='', + **kwargs): prediction_dict = {} binary_loss_type = { - LossType.F1_REWEIGHTED_LOSS, LossType.PAIR_WISE_LOSS, - LossType.BINARY_FOCAL_LOSS, LossType.PAIRWISE_FOCAL_LOSS, - LossType.LISTWISE_RANK_LOSS, LossType.PAIRWISE_HINGE_LOSS, - LossType.PAIRWISE_LOGISTIC_LOSS, LossType.BINARY_CROSS_ENTROPY_LOSS, - LossType.LISTWISE_DISTILL_LOSS + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.LISTWISE_RANK_LOSS, + LossType.PAIRWISE_HINGE_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.BINARY_CROSS_ENTROPY_LOSS, + LossType.LISTWISE_DISTILL_LOSS, } if loss_type in binary_loss_type: assert num_class == 1, 'num_class must be 1 when loss type is %s' % loss_type.name @@ -82,7 +88,15 @@ def _output_to_prediction_impl(self, prediction_dict['probs' + suffix] = probs[:, 1] elif loss_type == LossType.ZILN_LOSS: assert num_class == 3, 'num_class must be 3 when loss type is ZILN_LOSS' - probs, preds = zero_inflated_lognormal_pred(output) + max_log_clip_val = kwargs.get('max_log_clip_value', 20.0) + return_log = kwargs.get('return_log_pred_value', False) + max_sigma = kwargs.get('max_sigma', 5.0) + probs, preds = zero_inflated_lognormal_pred( + output, + max_sigma=max_sigma, + max_log_clip=max_log_clip_val, + return_log=return_log, + ) tf.summary.scalar('prediction/probs', tf.reduce_mean(probs)) tf.summary.scalar('prediction/y', tf.reduce_mean(preds)) prediction_dict['logits' + suffix] = output @@ -121,8 +135,20 @@ def _add_to_prediction_dict(self, output): self._prediction_dict.update(prediction_dict) else: for loss in self._losses: + kwargs = {} + if loss.loss_type == LossType.ZILN_LOSS: + loss_param = loss.WhichOneof('loss_param') + if loss_param is not None: + loss_param = getattr(loss, loss_param) + kwargs['max_log_clip_value'] = loss_param.max_log_clip_value + kwargs['return_log_pred_value'] = loss_param.return_log_pred_value + kwargs['max_sigma'] = loss_param.max_sigma prediction_dict = self._output_to_prediction_impl( - output, loss_type=loss.loss_type, num_class=self._num_class) + output, + loss_type=loss.loss_type, + num_class=self._num_class, + **kwargs, + ) self._prediction_dict.update(prediction_dict) def build_rtp_output_dict(self): @@ -144,11 +170,15 @@ def build_rtp_output_dict(self): if len(self._losses) > 0: loss_types = {loss.loss_type for loss in self._losses} binary_loss_set = { - LossType.CLASSIFICATION, LossType.F1_REWEIGHTED_LOSS, - LossType.PAIR_WISE_LOSS, LossType.BINARY_FOCAL_LOSS, - LossType.PAIRWISE_FOCAL_LOSS, LossType.PAIRWISE_LOGISTIC_LOSS, - LossType.JRC_LOSS, LossType.LISTWISE_DISTILL_LOSS, - LossType.LISTWISE_RANK_LOSS + LossType.CLASSIFICATION, + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.JRC_LOSS, + LossType.LISTWISE_DISTILL_LOSS, + LossType.LISTWISE_RANK_LOSS, } if loss_types & binary_loss_set: if 'probs' in self._prediction_dict: @@ -159,7 +189,9 @@ def build_rtp_output_dict(self): "expect 'probs' prediction, which is not found. Please check if" + ' build_predict_graph() is called.') elif loss_types & { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, + LossType.ZILN_LOSS, }: if 'y' in self._prediction_dict: forwarded = self._prediction_dict['y'] @@ -179,21 +211,28 @@ def build_rtp_output_dict(self): outputs['rank_predict'] = rank_predict return outputs - def _build_loss_impl(self, - loss_type, - label_name, - loss_weight=1.0, - num_class=1, - suffix='', - loss_name='', - loss_param=None): + def _build_loss_impl( + self, + loss_type, + label_name, + loss_weight=1.0, + num_class=1, + suffix='', + loss_name='', + loss_param=None, + ): loss_dict = {} binary_loss_type = { - LossType.F1_REWEIGHTED_LOSS, LossType.PAIR_WISE_LOSS, - LossType.BINARY_FOCAL_LOSS, LossType.PAIRWISE_FOCAL_LOSS, - LossType.LISTWISE_RANK_LOSS, LossType.PAIRWISE_HINGE_LOSS, - LossType.PAIRWISE_LOGISTIC_LOSS, LossType.JRC_LOSS, - LossType.LISTWISE_DISTILL_LOSS, LossType.ZILN_LOSS + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.LISTWISE_RANK_LOSS, + LossType.PAIRWISE_HINGE_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.JRC_LOSS, + LossType.LISTWISE_DISTILL_LOSS, + LossType.ZILN_LOSS, } if loss_type in { LossType.CLASSIFICATION, LossType.BINARY_CROSS_ENTROPY_LOSS @@ -212,8 +251,10 @@ def _build_loss_impl(self, else: raise ValueError('invalid loss type: %s' % LossType.Name(loss_type)) - tf.summary.scalar('labels/%s' % label_name, - tf.reduce_mean(tf.to_float(self._labels[label_name]))) + tf.summary.scalar( + 'labels/%s' % label_name, + tf.reduce_mean(tf.to_float(self._labels[label_name])), + ) kwargs = {'loss_name': loss_name} if loss_param is not None: if hasattr(loss_param, 'session_name'): @@ -225,7 +266,8 @@ def _build_loss_impl(self, loss_weight, num_class, loss_param=loss_param, - **kwargs) + **kwargs, + ) return loss_dict def build_loss_graph(self): @@ -236,7 +278,8 @@ def build_loss_graph(self): self._loss_type, label_name=self._label_name, loss_weight=self._sample_weight, - num_class=self._num_class) + num_class=self._num_class, + ) else: strategy = self._base_model_config.loss_weight_strategy loss_weight = [1.0] @@ -253,17 +296,22 @@ def build_loss_graph(self): loss_weight=self._sample_weight, num_class=self._num_class, loss_name=loss.loss_name, - loss_param=loss_param) + loss_param=loss_param, + ) for loss_name, loss_value in loss_ops.items(): if strategy == self._base_model_config.Fixed: loss_dict[loss_name] = loss_value * loss.weight elif strategy == self._base_model_config.Uncertainty: if loss.learn_loss_weight: uncertainty = tf.Variable( - 0, name='%s_loss_weight' % loss_name, dtype=tf.float32) + 0, + name='%s_loss_weight' % loss_name, + dtype=tf.float32, + ) tf.summary.scalar('%s_uncertainty' % loss_name, uncertainty) if loss.loss_type in { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, }: loss_dict[loss_name] = 0.5 * tf.exp( -uncertainty) * loss_value + 0.5 * uncertainty @@ -293,14 +341,20 @@ def _build_metric_impl(self, suffix=''): if not isinstance(loss_type, set): loss_type = {loss_type} - from easy_rec.python.core.easyrec_metrics import metrics_tf from easy_rec.python.core import metrics as metrics_lib + from easy_rec.python.core.easyrec_metrics import metrics_tf + binary_loss_set = { - LossType.CLASSIFICATION, LossType.F1_REWEIGHTED_LOSS, - LossType.PAIR_WISE_LOSS, LossType.BINARY_FOCAL_LOSS, - LossType.PAIRWISE_FOCAL_LOSS, LossType.PAIRWISE_LOGISTIC_LOSS, - LossType.JRC_LOSS, LossType.LISTWISE_DISTILL_LOSS, - LossType.LISTWISE_RANK_LOSS, LossType.ZILN_LOSS + LossType.CLASSIFICATION, + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.JRC_LOSS, + LossType.LISTWISE_DISTILL_LOSS, + LossType.LISTWISE_RANK_LOSS, + LossType.ZILN_LOSS, } metric_dict = {} if metric.WhichOneof('metric') == 'auc': @@ -310,13 +364,15 @@ def _build_metric_impl(self, metric_dict['auc' + suffix] = metrics_tf.auc( label, self._prediction_dict['probs' + suffix], - num_thresholds=metric.auc.num_thresholds) + num_thresholds=metric.auc.num_thresholds, + ) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) metric_dict['auc' + suffix] = metrics_tf.auc( label, self._prediction_dict['probs' + suffix][:, 1], - num_thresholds=metric.auc.num_thresholds) + num_thresholds=metric.auc.num_thresholds, + ) else: raise ValueError('Wrong class number') elif metric.WhichOneof('metric') == 'gauc': @@ -332,14 +388,16 @@ def _build_metric_impl(self, label, self._prediction_dict['probs' + suffix], uids=uids, - reduction=metric.gauc.reduction) + reduction=metric.gauc.reduction, + ) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) metric_dict['gauc' + suffix] = metrics_lib.gauc( label, self._prediction_dict['probs' + suffix][:, 1], uids=self._feature_dict[metric.gauc.uid_field], - reduction=metric.gauc.reduction) + reduction=metric.gauc.reduction, + ) else: raise ValueError('Wrong class number') elif metric.WhichOneof('metric') == 'session_auc': @@ -350,14 +408,16 @@ def _build_metric_impl(self, label, self._prediction_dict['probs' + suffix], session_ids=self._feature_dict[metric.session_auc.session_id_field], - reduction=metric.session_auc.reduction) + reduction=metric.session_auc.reduction, + ) elif num_class == 2: label = tf.to_int64(self._labels[label_name]) metric_dict['session_auc' + suffix] = metrics_lib.session_auc( label, self._prediction_dict['probs' + suffix][:, 1], session_ids=self._feature_dict[metric.session_auc.session_id_field], - reduction=metric.session_auc.reduction) + reduction=metric.session_auc.reduction, + ) else: raise ValueError('Wrong class number') elif metric.WhichOneof('metric') == 'max_f1': @@ -377,12 +437,16 @@ def _build_metric_impl(self, assert num_class > 1 label = tf.to_int64(self._labels[label_name]) metric_dict['recall_at_topk' + suffix] = metrics_tf.recall_at_k( - label, self._prediction_dict['logits' + suffix], - metric.recall_at_topk.topk) + label, + self._prediction_dict['logits' + suffix], + metric.recall_at_topk.topk, + ) elif metric.WhichOneof('metric') == 'mean_absolute_error': label = tf.to_float(self._labels[label_name]) if loss_type & { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, + LossType.ZILN_LOSS, }: metric_dict['mean_absolute_error' + suffix] = metrics_tf.mean_absolute_error( @@ -396,7 +460,9 @@ def _build_metric_impl(self, elif metric.WhichOneof('metric') == 'mean_squared_error': label = tf.to_float(self._labels[label_name]) if loss_type & { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, + LossType.ZILN_LOSS, }: metric_dict['mean_squared_error' + suffix] = metrics_tf.mean_squared_error( @@ -410,7 +476,9 @@ def _build_metric_impl(self, elif metric.WhichOneof('metric') == 'root_mean_squared_error': label = tf.to_float(self._labels[label_name]) if loss_type & { - LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS, LossType.ZILN_LOSS + LossType.L2_LOSS, + LossType.SIGMOID_L2_LOSS, + LossType.ZILN_LOSS, }: metric_dict['root_mean_squared_error' + suffix] = metrics_tf.root_mean_squared_error( @@ -439,15 +507,20 @@ def build_metric_graph(self, eval_config): metric, loss_type=loss_types, label_name=self._label_name, - num_class=self._num_class)) + num_class=self._num_class, + )) return self._metric_dict def _get_outputs_impl(self, loss_type, num_class=1, suffix=''): binary_loss_set = { - LossType.F1_REWEIGHTED_LOSS, LossType.PAIR_WISE_LOSS, - LossType.BINARY_FOCAL_LOSS, LossType.PAIRWISE_FOCAL_LOSS, - LossType.LISTWISE_RANK_LOSS, LossType.PAIRWISE_HINGE_LOSS, - LossType.PAIRWISE_LOGISTIC_LOSS, LossType.LISTWISE_DISTILL_LOSS + LossType.F1_REWEIGHTED_LOSS, + LossType.PAIR_WISE_LOSS, + LossType.BINARY_FOCAL_LOSS, + LossType.PAIRWISE_FOCAL_LOSS, + LossType.LISTWISE_RANK_LOSS, + LossType.PAIRWISE_HINGE_LOSS, + LossType.PAIRWISE_LOGISTIC_LOSS, + LossType.LISTWISE_DISTILL_LOSS, } if loss_type in binary_loss_set: return ['probs' + suffix, 'logits' + suffix] @@ -460,9 +533,13 @@ def _get_outputs_impl(self, loss_type, num_class=1, suffix=''): return ['probs' + suffix, 'logits' + suffix] else: return [ - 'y' + suffix, 'probs' + suffix, 'logits' + suffix, - 'probs' + suffix + '_y', 'logits' + suffix + '_y', - 'probs' + suffix + '_1', 'logits' + suffix + '_1' + 'y' + suffix, + 'probs' + suffix, + 'logits' + suffix, + 'probs' + suffix + '_y', + 'logits' + suffix + '_y', + 'probs' + suffix + '_1', + 'logits' + suffix + '_1', ] elif loss_type in [LossType.L2_LOSS, LossType.SIGMOID_L2_LOSS]: return ['y' + suffix] diff --git a/easy_rec/python/model/rocket_launching.py b/easy_rec/python/model/rocket_launching.py index aea29bf52..d98b8b854 100755 --- a/easy_rec/python/model/rocket_launching.py +++ b/easy_rec/python/model/rocket_launching.py @@ -24,8 +24,8 @@ def __init__(self, is_training=False): super(RocketLaunching, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'rocket_launching', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof('model') == 'rocket_launching', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.rocket_launching assert isinstance(self._model_config, RocketLaunchingConfig) if self._labels is not None: @@ -61,11 +61,19 @@ def feature_based_sim(self, feature_based_distillation, i, j): def build_predict_graph(self): self.hidden_layer_feature_output = self._model_config.feature_based_distillation if self._model_config.HasField('share_dnn'): - share_dnn_layer = dnn.DNN(self._model_config.share_dnn, self._l2_reg, - 'share_dnn', self._is_training) + share_dnn_layer = dnn.DNN( + self._model_config.share_dnn, + self._l2_reg, + 'share_dnn', + self._is_training, + ) share_feature = share_dnn_layer(self._features) - booster_dnn_layer = dnn.DNN(self._model_config.booster_dnn, self._l2_reg, - 'booster_dnn', self._is_training) + booster_dnn_layer = dnn.DNN( + self._model_config.booster_dnn, + self._l2_reg, + 'booster_dnn', + self._is_training, + ) light_dnn_layer = dnn.DNN(self._model_config.light_dnn, self._l2_reg, 'light_dnn', self._is_training) if self._model_config.HasField('share_dnn'): @@ -86,32 +94,37 @@ def build_predict_graph(self): self.booster_feature['hidden_layer_end'], self._num_class, kernel_regularizer=self._l2_reg, - name='booster_output') + name='booster_output', + ) light_out = tf.layers.dense( self.light_feature['hidden_layer_end'], self._num_class, kernel_regularizer=self._l2_reg, - name='light_output') + name='light_output', + ) else: booster_out = tf.layers.dense( self.booster_feature, self._num_class, kernel_regularizer=self._l2_reg, - name='booster_output') + name='booster_output', + ) light_out = tf.layers.dense( self.light_feature, self._num_class, kernel_regularizer=self._l2_reg, - name='light_output') + name='light_output', + ) self._prediction_dict.update( self._output_to_prediction_impl( booster_out, self._loss_type, num_class=self._num_class, - suffix='_booster')) + suffix='_booster', + )) self._prediction_dict.update( self._output_to_prediction_impl( light_out, @@ -147,7 +160,8 @@ def build_loss_graph(self): label_name=self._label_name, loss_weight=self._sample_weight, num_class=self._num_class, - suffix='_booster')) + suffix='_booster', + )) self._loss_dict.update( self._build_loss_impl( @@ -155,7 +169,8 @@ def build_loss_graph(self): label_name=self._label_name, loss_weight=self._sample_weight, num_class=self._num_class, - suffix='_light')) + suffix='_light', + )) booster_logits_no_grad = tf.stop_gradient(logits_booster) @@ -163,7 +178,8 @@ def build_loss_graph(self): LossType.L2_LOSS, label=booster_logits_no_grad, pred=logits_light, - loss_weight=self._sample_weight) + loss_weight=self._sample_weight, + ) if self._model_config.feature_based_distillation: for key, value in self._prediction_dict.items(): @@ -182,14 +198,16 @@ def build_metric_graph(self, eval_config): loss_type=LossType.CLASSIFICATION, label_name=self._label_name, num_class=self._num_class, - suffix='_light')) + suffix='_light', + )) metric_dict.update( self._build_metric_impl( metric, loss_type=LossType.CLASSIFICATION, label_name=self._label_name, num_class=self._num_class, - suffix='_booster')) + suffix='_booster', + )) return metric_dict def get_outputs(self): diff --git a/easy_rec/python/model/simple_multi_task.py b/easy_rec/python/model/simple_multi_task.py index 05dd7a773..8215a7b4c 100644 --- a/easy_rec/python/model/simple_multi_task.py +++ b/easy_rec/python/model/simple_multi_task.py @@ -22,8 +22,8 @@ def __init__(self, super(SimpleMultiTask, self).__init__(model_config, feature_configs, features, labels, is_training) - assert self._model_config.WhichOneof('model') == 'simple_multi_task', \ - 'invalid model config: %s' % self._model_config.WhichOneof('model') + assert self._model_config.WhichOneof('model') == 'simple_multi_task', ( + 'invalid model config: %s' % self._model_config.WhichOneof('model')) self._model_config = self._model_config.simple_multi_task assert isinstance(self._model_config, SimpleMultiTaskConfig) @@ -41,13 +41,15 @@ def build_predict_graph(self): task_tower_cfg.dnn, self._l2_reg, name=tower_name, - is_training=self._is_training) + is_training=self._is_training, + ) task_fea = task_dnn(self._features) task_output = tf.layers.dense( inputs=task_fea, units=task_tower_cfg.num_class, kernel_regularizer=self._l2_reg, - name='dnn_output_%d' % i) + name='dnn_output_%d' % i, + ) tower_outputs[tower_name] = task_output self._add_to_prediction_dict(tower_outputs) diff --git a/easy_rec/python/model/uniter.py b/easy_rec/python/model/uniter.py index 40dfc8cb1..2fd8c9c8b 100644 --- a/easy_rec/python/model/uniter.py +++ b/easy_rec/python/model/uniter.py @@ -30,9 +30,13 @@ def __init__(self, assert self._model_config.WhichOneof('model') == 'uniter', ( 'invalid model config: %s' % self._model_config.WhichOneof('model')) - self._uniter_layer = uniter.Uniter(model_config, feature_configs, features, - self._model_config.uniter.config, - self._input_layer) + self._uniter_layer = uniter.Uniter( + model_config, + feature_configs, + features, + self._model_config.uniter.config, + self._input_layer, + ) self._model_config = self._model_config.uniter def build_predict_graph(self): diff --git a/easy_rec/python/model/wide_and_deep.py b/easy_rec/python/model/wide_and_deep.py index 48b620bd7..78b538523 100755 --- a/easy_rec/python/model/wide_and_deep.py +++ b/easy_rec/python/model/wide_and_deep.py @@ -23,8 +23,10 @@ def __init__(self, is_training=False): super(WideAndDeep, self).__init__(model_config, feature_configs, features, labels, is_training) - assert model_config.WhichOneof('model') == 'wide_and_deep', \ - 'invalid model config: %s' % model_config.WhichOneof('model') + assert model_config.WhichOneof( + 'model' + ) == 'wide_and_deep', 'invalid model config: %s' % model_config.WhichOneof( + 'model') self._model_config = model_config.wide_and_deep assert isinstance(self._model_config, WideAndDeepConfig) assert self._input_layer.has_group('wide') @@ -59,8 +61,12 @@ def build_predict_graph(self): print('wide_deep has_final_dnn layers = %d' % has_final) if has_final: all_fea = tf.concat([wide_fea, deep_fea], axis=1) - final_layer = dnn.DNN(self._model_config.final_dnn, self._l2_reg, - 'final_dnn', self._is_training) + final_layer = dnn.DNN( + self._model_config.final_dnn, + self._l2_reg, + 'final_dnn', + self._is_training, + ) all_fea = final_layer(all_fea) output = tf.layers.dense( all_fea, @@ -72,7 +78,8 @@ def build_predict_graph(self): deep_fea, self._num_class, kernel_regularizer=self._l2_reg, - name='deep_out') + name='deep_out', + ) output = deep_out + wide_fea self._add_to_prediction_dict(output) @@ -90,17 +97,18 @@ def get_grouped_vars(self, opt_num): Return: list of list of variables. """ - assert opt_num <= 3, 'could only support 2 or 3 optimizers, ' + \ - 'if opt_num = 2, one for the wide , and one for the others, ' + \ - 'if opt_num = 3, one for the wide, second for the deep embeddings, ' + \ - 'and third for the other layers.' + assert opt_num <= 3, ( + 'could only support 2 or 3 optimizers, ' + + 'if opt_num = 2, one for the wide , and one for the others, ' + + 'if opt_num = 3, one for the wide, second for the deep embeddings, ' + + 'and third for the other layers.') if opt_num == 2: wide_vars = [] deep_vars = [] for tmp_var in tf.trainable_variables(): - if tmp_var.name.startswith('input_layer') and \ - (not tmp_var.name.startswith('input_layer_1')): + if tmp_var.name.startswith('input_layer') and ( + not tmp_var.name.startswith('input_layer_1')): wide_vars.append(tmp_var) else: deep_vars.append(tmp_var) @@ -110,8 +118,8 @@ def get_grouped_vars(self, opt_num): embedding_vars = [] deep_vars = [] for tmp_var in tf.trainable_variables(): - if tmp_var.name.startswith('input_layer') and \ - (not tmp_var.name.startswith('input_layer_1')): + if tmp_var.name.startswith('input_layer') and ( + not tmp_var.name.startswith('input_layer_1')): wide_vars.append(tmp_var) elif tmp_var.name.startswith( 'input_layer') or '/embedding_weights' in tmp_var.name: diff --git a/easy_rec/python/ops/gen_kafka_ops.py b/easy_rec/python/ops/gen_kafka_ops.py index 16bba500d..4948d377e 100644 --- a/easy_rec/python/ops/gen_kafka_ops.py +++ b/easy_rec/python/ops/gen_kafka_ops.py @@ -33,16 +33,18 @@ @tf_export('io_kafka_dataset_v2') -def io_kafka_dataset_v2(topics, - servers, - group, - eof, - timeout, - config_global, - config_topic, - message_key, - message_offset, - name=None): +def io_kafka_dataset_v2( + topics, + servers, + group, + eof, + timeout, + config_global, + config_topic, + message_key, + message_offset, + name=None, +): """Creates a dataset that emits the messages of one or more Kafka topics. Args: @@ -82,20 +84,23 @@ def io_kafka_dataset_v2(topics, config_topic=config_topic, message_key=message_key, message_offset=message_offset, - name=name) - - -def io_kafka_dataset_eager_fallback(topics, - servers, - group, - eof, - timeout, - config_global, - config_topic, - message_key, - message_offset, - name=None, - ctx=None): + name=name, + ) + + +def io_kafka_dataset_eager_fallback( + topics, + servers, + group, + eof, + timeout, + config_global, + config_topic, + message_key, + message_offset, + name=None, + ctx=None, +): """This is the slowpath function for Eager mode. This is for function io_kafka_dataset @@ -111,8 +116,15 @@ def io_kafka_dataset_eager_fallback(topics, message_key = _ops.convert_to_tensor(message_key, _dtypes.bool) message_offset = _ops.convert_to_tensor(message_offset, _dtypes.bool) _inputs_flat = [ - topics, servers, group, eof, timeout, config_global, config_topic, - message_key, message_offset + topics, + servers, + group, + eof, + timeout, + config_global, + config_topic, + message_key, + message_offset, ] _attrs = None _result = _execute.execute( @@ -124,7 +136,7 @@ def io_kafka_dataset_eager_fallback(topics, name=name) _execute.record_gradient('IOKafkaDataset', _inputs_flat, _attrs, _result, name) - _result, = _result + (_result,) = _result return _result @@ -150,14 +162,21 @@ def io_write_kafka_v2(message, topic, servers, name=None): _attrs = None _execute.record_gradient('IOWriteKafka', _inputs_flat, _attrs, _result, name) - _result, = _result + (_result,) = _result return _result else: try: _result = _pywrap_tensorflow.TFE_Py_FastPathExecute( - _ctx._context_handle, _ctx._eager_context.device_name, 'IOWriteKafka', - name, _ctx._post_execution_callbacks, message, topic, servers) + _ctx._context_handle, + _ctx._eager_context.device_name, + 'IOWriteKafka', + name, + _ctx._post_execution_callbacks, + message, + topic, + servers, + ) return _result except _core._FallbackException: return io_write_kafka_eager_fallback( @@ -189,5 +208,5 @@ def io_write_kafka_eager_fallback(message, topic, servers, name=None, ctx=None): ctx=_ctx, name=name) _execute.record_gradient('IOWriteKafka', _inputs_flat, _attrs, _result, name) - _result, = _result + (_result,) = _result return _result diff --git a/easy_rec/python/ops/gen_str_avx_op.py b/easy_rec/python/ops/gen_str_avx_op.py index d022d52cb..ba57d41f1 100644 --- a/easy_rec/python/ops/gen_str_avx_op.py +++ b/easy_rec/python/ops/gen_str_avx_op.py @@ -20,9 +20,10 @@ def str_split_by_chr(input_str, sep, skip_empty): if constant.has_avx_str_split() and str_avx_op is not None: - assert len(sep) == 1, \ - 'invalid data_config.separator(%s) len(%d) != 1' % ( - sep, len(sep)) + assert len(sep) == 1, 'invalid data_config.separator(%s) len(%d) != 1' % ( + sep, + len(sep), + ) return str_avx_op.avx512_string_split(input_str, sep, skip_empty=skip_empty) else: return string_ops.string_split(input_str, sep, skip_empty=skip_empty) diff --git a/easy_rec/python/predict.py b/easy_rec/python/predict.py index ced1a7573..3362ba7b4 100644 --- a/easy_rec/python/predict.py +++ b/easy_rec/python/predict.py @@ -18,14 +18,16 @@ from easy_rec.python.utils import numpy_utils from easy_rec.python.utils.hive_utils import HiveUtils -from easy_rec.python.inference.hive_parquet_predictor import HiveParquetPredictor # NOQA +from easy_rec.python.inference.hive_parquet_predictor import ( # NOQA + HiveParquetPredictor,) if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + level=logging.INFO, +) tf.app.flags.DEFINE_string('input_path', None, 'predict data path') tf.app.flags.DEFINE_string('output_path', None, 'path to save predict result') @@ -37,19 +39,26 @@ 'Path to pipeline config ' 'file.') tf.app.flags.DEFINE_string( - 'checkpoint_path', None, 'checkpoint to be evaled ' + 'checkpoint_path', + None, + 'checkpoint to be evaled ' ' if not specified, use the latest checkpoint in ' - 'train_config.model_dir') + 'train_config.model_dir', +) tf.app.flags.DEFINE_string('model_dir', None, help='will update the model_dir') # predict by saved_model tf.app.flags.DEFINE_string('saved_model_dir', None, help='save model dir') tf.app.flags.DEFINE_string( - 'reserved_cols', 'ALL_COLUMNS', - 'columns to keep from input table, they are separated with ,') + 'reserved_cols', + 'ALL_COLUMNS', + 'columns to keep from input table, they are separated with ,', +) tf.app.flags.DEFINE_string( - 'output_cols', 'ALL_COLUMNS', - 'output columns, such as: score float. multiple columns are separated by ,') + 'output_cols', + 'ALL_COLUMNS', + 'output columns, such as: score float. multiple columns are separated by ,', +) tf.app.flags.DEFINE_string('output_sep', chr(1), 'separator of predict result file') tf.app.flags.DEFINE_string('selected_cols', None, '') @@ -69,7 +78,6 @@ def get_input_type(input_type, data_config): def main(argv): - if FLAGS.saved_model_dir: logging.info('Predict by saved_model.') if FLAGS.pipeline_config_path: @@ -84,8 +92,8 @@ def main(argv): if input_type in [data_config.HiveParquetInput, data_config.HiveInput]: all_cols, all_col_types = HiveUtils( data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input).get_all_cols( - FLAGS.input_path) + hive_config=pipeline_config.hive_train_input, + ).get_all_cols(FLAGS.input_path) if input_type == DatasetConfig.HiveParquetInput: predictor = HiveParquetPredictor( FLAGS.saved_model_dir, @@ -94,7 +102,8 @@ def main(argv): hive_config=pipeline_config.hive_train_input, output_sep=FLAGS.output_sep, all_cols=all_cols, - all_col_types=all_col_types) + all_col_types=all_col_types, + ) else: predictor = HivePredictor( FLAGS.saved_model_dir, @@ -103,7 +112,8 @@ def main(argv): hive_config=pipeline_config.hive_train_input, output_sep=FLAGS.output_sep, all_cols=all_cols, - all_col_types=all_col_types) + all_col_types=all_col_types, + ) elif input_type in [data_config.ParquetInput, data_config.ParquetInputV2]: predictor_cls = ParquetPredictor if input_type == data_config.ParquetInputV2: @@ -115,7 +125,8 @@ def main(argv): fg_json_path=FLAGS.fg_json_path, selected_cols=FLAGS.selected_cols, output_sep=FLAGS.output_sep, - pipeline_config=pipeline_config) + pipeline_config=pipeline_config, + ) elif input_type == data_config.CSVInput: predictor = CSVPredictor( FLAGS.saved_model_dir, @@ -124,7 +135,8 @@ def main(argv): ds_vector_recall=FLAGS.ds_vector_recall, fg_json_path=FLAGS.fg_json_path, selected_cols=FLAGS.selected_cols, - output_sep=FLAGS.output_sep) + output_sep=FLAGS.output_sep, + ) else: assert False, 'invalid input type: %s' % input_class_map_r[input_type] @@ -144,7 +156,8 @@ def main(argv): output_cols=FLAGS.output_cols, batch_size=FLAGS.batch_size, slice_id=task_index, - slice_num=worker_num) + slice_num=worker_num, + ) else: logging.info('Predict by checkpoint_path.') assert FLAGS.model_dir or FLAGS.pipeline_config_path, 'At least one of model_dir and pipeline_config_path exists.' diff --git a/easy_rec/python/protos/loss.proto b/easy_rec/python/protos/loss.proto index 4416111a8..c5dfed839 100644 --- a/easy_rec/python/protos/loss.proto +++ b/easy_rec/python/protos/loss.proto @@ -44,9 +44,20 @@ message Loss { PairwiseHingeLoss pairwise_hinge_loss = 110; ListwiseRankLoss listwise_rank_loss = 111; ListwiseDistillLoss listwise_distill_loss = 112; + ZILNLoss ziln_loss = 113; } }; +message ZILNLoss { + optional float mu_regularization = 1 [default = 0.01]; + optional float sigma_regularization = 2 [default = 0.01]; + optional float max_log_clip_value = 3 [default = 20.0]; + optional float max_sigma = 4 [default = 5.0]; + optional bool return_log_pred_value = 5 [default = false]; + optional float classification_weight = 6 [default = 1.0]; + optional float regression_weight = 7 [default = 1.0]; +} + message SoftmaxCrossEntropyWithNegativeMining { required uint32 num_negative_samples = 1; required float margin = 2 [default = 0]; diff --git a/easy_rec/python/test/csv_input_test.py b/easy_rec/python/test/csv_input_test.py index ae0793fa5..0b2e41c64 100644 --- a/easy_rec/python/test/csv_input_test.py +++ b/easy_rec/python/test/csv_input_test.py @@ -83,7 +83,8 @@ def test_csv_data(self): session_config = tf.ConfigProto( gpu_options=gpu_options, allow_soft_placement=True, - log_device_placement=False) + log_device_placement=False, + ) with self.test_session(config=session_config) as sess: sess.run(init_op) feature_dict, label_dict = sess.run([features, labels]) @@ -205,7 +206,8 @@ def test_csv_data_flt_to_str(self): session_config = tf.ConfigProto( gpu_options=gpu_options, allow_soft_placement=True, - log_device_placement=False) + log_device_placement=False, + ) with self.test_session(config=session_config) as sess: sess.run(init_op) feature_dict, label_dict = sess.run([features, labels]) @@ -263,13 +265,16 @@ def test_csv_input_ex(self): session_config = tf.ConfigProto( gpu_options=gpu_options, allow_soft_placement=True, - log_device_placement=False) + log_device_placement=False, + ) with self.test_session(config=session_config) as sess: sess.run(init_op) feature_dict, label_dict = sess.run([features, labels]) - @unittest.skipIf('AVX_TEST' not in os.environ, - 'Only execute when avx512 instructions are supported') + @unittest.skipIf( + 'AVX_TEST' not in os.environ, + 'Only execute when avx512 instructions are supported', + ) @RunAsSubprocess def test_csv_input_ex_avx(self): constant.enable_avx_str_split() @@ -330,7 +335,8 @@ def test_csv_data_ignore_error(self): session_config = tf.ConfigProto( gpu_options=gpu_options, allow_soft_placement=True, - log_device_placement=False) + log_device_placement=False, + ) with self.test_session(config=session_config) as sess: sess.run(init_op) feature_dict, label_dict = sess.run([features, labels]) diff --git a/easy_rec/python/test/dh_local_run.py b/easy_rec/python/test/dh_local_run.py index be514d382..f243c7d84 100644 --- a/easy_rec/python/test/dh_local_run.py +++ b/easy_rec/python/test/dh_local_run.py @@ -8,11 +8,12 @@ from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare -from easy_rec.python.test.odps_test_util import OdpsOSSConfig -from easy_rec.python.test.odps_test_util import delete_oss_path -from easy_rec.python.test.odps_test_util import get_oss_bucket from easy_rec.python.utils import test_utils +from easy_rec.python.test.odps_test_util import ( # NOQA + OdpsOSSConfig, delete_oss_path, get_oss_bucket, +) + logging.basicConfig( level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') @@ -37,8 +38,10 @@ def test_datahub_train_eval(self): odps_cmd = OdpsCommand(odps_oss_config) self._success = test_utils.test_datahub_train_eval( - '%s/configs/deepfm.config' % odps_oss_config.temp_dir, odps_oss_config, - self._test_dir) + '%s/configs/deepfm.config' % odps_oss_config.temp_dir, + odps_oss_config, + self._test_dir, + ) odps_cmd.run_list(end) self.assertTrue(self._success) @@ -89,7 +92,7 @@ def test_datahub_train_eval(self): prepare(odps_oss_config) start = [ 'deep_fm/create_external_deepfm_table.sql', - 'deep_fm/create_inner_deepfm_table.sql' + 'deep_fm/create_inner_deepfm_table.sql', ] end = ['deep_fm/drop_table.sql'] odps_cmd = OdpsCommand(odps_oss_config) @@ -97,8 +100,12 @@ def test_datahub_train_eval(self): odps_oss_config.init_dh_and_odps() tf.test.main() # delete oss path - bucket = get_oss_bucket(odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name) + bucket = get_oss_bucket( + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, + ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) # delete tmp shutil.rmtree(odps_oss_config.temp_dir) diff --git a/easy_rec/python/test/embed_test.py b/easy_rec/python/test/embed_test.py index e499b29d0..07e1bdede 100644 --- a/easy_rec/python/test/embed_test.py +++ b/easy_rec/python/test/embed_test.py @@ -28,7 +28,7 @@ def test_raw_embed(self): # [7, 8 ], # [9, 10] # ] - feature_config_str = ''' + feature_config_str = """ input_names: 'field1' feature_type: RawFeature initializer { @@ -40,11 +40,11 @@ def test_raw_embed(self): raw_input_dim: 5 embedding_dim: 2 combiner: 'sum' - ''' + """ feature_config = FeatureConfig() text_format.Merge(feature_config_str, feature_config) - data_config_str = ''' + data_config_str = """ input_fields { input_name: 'clk' input_type: INT32 @@ -57,7 +57,7 @@ def test_raw_embed(self): } label_fields: 'clk' batch_size: 1 - ''' + """ data_config = DatasetConfig() text_format.Merge(data_config_str, data_config) @@ -93,7 +93,7 @@ def test_seq_multi_embed(self): # [7, 8 ], # [9, 10] # ] - feature_config_str = ''' + feature_config_str = """ input_names: 'field1' feature_type: SequenceFeature initializer { @@ -106,11 +106,11 @@ def test_seq_multi_embed(self): embedding_dim: 2 num_buckets: 5 combiner: 'mean' - ''' + """ feature_config = FeatureConfig() text_format.Merge(feature_config_str, feature_config) - data_config_str = ''' + data_config_str = """ input_fields { input_name: 'clk' input_type: INT32 @@ -123,7 +123,7 @@ def test_seq_multi_embed(self): } label_fields: 'clk' batch_size: 1 - ''' + """ data_config = DatasetConfig() text_format.Merge(data_config_str, data_config) @@ -136,8 +136,8 @@ def test_seq_multi_embed(self): wide_and_deep_dict = {'field1': WideOrDeep.DEEP} fc_parser = FeatureColumnParser(feature_configs, wide_and_deep_dict) builder = feature_column._LazyBuilder(field_dict) - hist_embedding, hist_seq_len = \ - fc_parser.sequence_columns['field1']._get_sequence_dense_tensor(builder) + hist_embedding, hist_seq_len = fc_parser.sequence_columns[ + 'field1']._get_sequence_dense_tensor(builder) init = tf.initialize_all_variables() with tf.Session() as sess: diff --git a/easy_rec/python/test/emr_run.py b/easy_rec/python/test/emr_run.py index 305190544..0a514ccaf 100644 --- a/easy_rec/python/test/emr_run.py +++ b/easy_rec/python/test/emr_run.py @@ -11,11 +11,12 @@ from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare -from easy_rec.python.test.odps_test_util import OdpsOSSConfig -from easy_rec.python.test.odps_test_util import delete_oss_path -from easy_rec.python.test.odps_test_util import get_oss_bucket from easy_rec.python.utils import test_utils +from easy_rec.python.test.odps_test_util import ( # NOQA + OdpsOSSConfig, delete_oss_path, get_oss_bucket, +) + logging.basicConfig( level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') @@ -40,7 +41,7 @@ def tearDown(self): def test_deepfm_train_eval_export(self): start = [ 'deep_fm/create_external_deepfm_table.sql', - 'deep_fm/create_inner_deepfm_table.sql' + 'deep_fm/create_inner_deepfm_table.sql', ] end = ['deep_fm/drop_table.sql'] odps_cmd = OdpsCommand(odps_oss_config) @@ -48,19 +49,22 @@ def test_deepfm_train_eval_export(self): self._success = test_utils.test_hdfs_train_eval( '%s/configs/deepfm.config' % odps_oss_config.temp_dir, '%s/yaml_config/train.paitf.yaml' % odps_oss_config.temp_dir, - self._test_hdfs_dir) + self._test_hdfs_dir, + ) self.assertTrue(self._success) self._success = test_utils.test_hdfs_eval( '%s/configs/deepfm_eval_pipeline.config' % odps_oss_config.temp_dir, '%s/yaml_config/eval.tf.yaml' % odps_oss_config.temp_dir, - self._test_hdfs_dir) + self._test_hdfs_dir, + ) self.assertTrue(self._success) self._success = test_utils.test_hdfs_export( '%s/configs/deepfm_eval_pipeline.config' % odps_oss_config.temp_dir, '%s/yaml_config/export.tf.yaml' % odps_oss_config.temp_dir, - self._test_hdfs_dir) + self._test_hdfs_dir, + ) self.assertTrue(self._success) odps_cmd.run_list(end) @@ -112,8 +116,12 @@ def test_deepfm_train_eval_export(self): prepare(odps_oss_config) tf.test.main() # delete oss path - bucket = get_oss_bucket(odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name) + bucket = get_oss_bucket( + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, + ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) # delete tmp shutil.rmtree(odps_oss_config.temp_dir) diff --git a/easy_rec/python/test/eval_metric_test.py b/easy_rec/python/test/eval_metric_test.py index 22e019b6a..186cd8ced 100644 --- a/easy_rec/python/test/eval_metric_test.py +++ b/easy_rec/python/test/eval_metric_test.py @@ -21,6 +21,7 @@ def setUp(self): @RunAsSubprocess def test_max_f1(self): from easy_rec.python.core.metrics import max_f1 + labels = tf.constant([1, 0, 0, 1], dtype=tf.int32) probs = tf.constant([0.9, 0.8, 0.7, 0.6], dtype=tf.float32) f1, f1_update_op = max_f1(labels, probs) @@ -34,6 +35,7 @@ def test_max_f1(self): @RunAsSubprocess def test_gauc_all_negative_label(self): from easy_rec.python.core.metrics import gauc + labels = tf.constant([0, 0, 0, 0], dtype=tf.int32) probs = tf.constant([0.9, 0.8, 0.7, 0.6], dtype=tf.float32) uids = tf.constant([1, 1, 1, 1], dtype=tf.int32) @@ -43,13 +45,15 @@ def test_gauc_all_negative_label(self): score = sess.run(value_op) self.assertAlmostEqual(score, 0.0) - @parameterized.named_parameters( - [['_reduction_mean', 'mean', 0.5833333], - ['_reduction_mean_by_sample_num', 'mean_by_sample_num', 0.5925926], - ['_reduction_mean_by_positive_num', 'mean_by_positive_num', 0.6]]) + @parameterized.named_parameters([ + ['_reduction_mean', 'mean', 0.5833333], + ['_reduction_mean_by_sample_num', 'mean_by_sample_num', 0.5925926], + ['_reduction_mean_by_positive_num', 'mean_by_positive_num', 0.6], + ]) @RunAsSubprocess def test_gauc(self, reduction, expected): from easy_rec.python.core.metrics import gauc + labels = tf.placeholder(dtype=tf.int32, shape=(None,)) probs = tf.placeholder(dtype=tf.float32, shape=(None,)) uids = tf.placeholder(dtype=tf.int32, shape=(None,)) @@ -60,25 +64,29 @@ def test_gauc(self, reduction, expected): feed_dict={ labels: [1, 0, 1, 1, 0], probs: [0.9, 0.8, 0.7, 0.6, 0.5], - uids: [1, 1, 1, 1, 1] - }) + uids: [1, 1, 1, 1, 1], + }, + ) sess.run( update_op, feed_dict={ labels: [1, 0, 0, 1], probs: [0.9, 0.8, 0.7, 0.6], - uids: [2, 2, 2, 2] - }) + uids: [2, 2, 2, 2], + }, + ) score = sess.run(value_op) self.assertAlmostEqual(score, expected) - @parameterized.named_parameters( - [['_reduction_mean', 'mean', 0.5833333], - ['_reduction_mean_by_sample_num', 'mean_by_sample_num', 0.5925926], - ['_reduction_mean_by_positive_num', 'mean_by_positive_num', 0.6]]) + @parameterized.named_parameters([ + ['_reduction_mean', 'mean', 0.5833333], + ['_reduction_mean_by_sample_num', 'mean_by_sample_num', 0.5925926], + ['_reduction_mean_by_positive_num', 'mean_by_positive_num', 0.6], + ]) @RunAsSubprocess def test_session_auc(self, reduction, expected): from easy_rec.python.core.metrics import session_auc + labels = tf.placeholder(dtype=tf.int32, shape=(None,)) probs = tf.placeholder(dtype=tf.float32, shape=(None,)) session_ids = tf.placeholder(dtype=tf.int32, shape=(None,)) @@ -90,15 +98,17 @@ def test_session_auc(self, reduction, expected): feed_dict={ labels: [1, 0, 1, 1, 0], probs: [0.9, 0.8, 0.7, 0.6, 0.5], - session_ids: [1, 1, 1, 1, 1] - }) + session_ids: [1, 1, 1, 1, 1], + }, + ) sess.run( update_op, feed_dict={ labels: [1, 0, 0, 1], probs: [0.9, 0.8, 0.7, 0.6], - session_ids: [2, 2, 2, 2] - }) + session_ids: [2, 2, 2, 2], + }, + ) score = sess.run(value_op) self.assertAlmostEqual(score, expected) diff --git a/easy_rec/python/test/excel_convert_test.py b/easy_rec/python/test/excel_convert_test.py index a860d3277..3e83d18b5 100644 --- a/easy_rec/python/test/excel_convert_test.py +++ b/easy_rec/python/test/excel_convert_test.py @@ -21,14 +21,14 @@ def test_deepfm_convert(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'avazu_deepfm_excel.config') - convert_cmd = """ + convert_cmd = (""" python -m easy_rec.python.tools.create_config_from_excel --excel_path samples/excel_config/dwd_avazu_ctr_deepfm.xls --model_type deepfm --output_path %s --train_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv --eval_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv - """ % pipeline_config_path + """ % pipeline_config_path) proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() @@ -42,14 +42,14 @@ def test_multi_tower_convert(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'avazu_deepfm_excel.config') - convert_cmd = """ + convert_cmd = (""" python -m easy_rec.python.tools.create_config_from_excel --excel_path samples/excel_config/dwd_avazu_ctr_deepfm.xls --model_type multi_tower --output_path %s --train_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv --eval_input_path data/test/dwd_avazu_ctr_deepmodel_10w.csv - """ % pipeline_config_path + """ % pipeline_config_path) proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() diff --git a/easy_rec/python/test/export_test.py b/easy_rec/python/test/export_test.py index 23bff2890..1761df360 100644 --- a/easy_rec/python/test/export_test.py +++ b/easy_rec/python/test/export_test.py @@ -28,13 +28,15 @@ def tearDown(self): test_utils.set_gpu_id(None) @RunAsSubprocess - def _predict_and_check(self, - data_path, - saved_model_dir, - cmp_result, - keys=['probs'], - separator=',', - tol=1e-4): + def _predict_and_check( + self, + data_path, + saved_model_dir, + cmp_result, + keys=['probs'], + separator=',', + tol=1e-4, + ): predictor = Predictor(saved_model_dir) with open(data_path, 'r') as fin: inputs = [] @@ -51,9 +53,11 @@ def _predict_and_check(self, val0 = output_res[i][key] val1 = cmp_result[i][key] diff = np.max(np.abs(val0 - val1)) - assert diff < tol, \ - 'too much difference: %.6f for %s, tol=%.6f' \ - % (diff, key, tol) + assert diff < tol, 'too much difference: %.6f for %s, tol=%.6f' % ( + diff, + key, + tol, + ) def _extract_data(self, input_path, output_path, offset=1, separator=','): with open(input_path, 'r') as fin: @@ -85,20 +89,23 @@ def test_mmoe(self): self._export_test( 'samples/model_config/mmoe_on_taobao.config', functools.partial(self._extract_data, offset=2), - keys=['probs_ctr', 'probs_cvr']) + keys=['probs_ctr', 'probs_cvr'], + ) def test_fg(self): self._export_test( 'samples/model_config/taobao_fg.config', self._extract_rtp_data, - separator='') + separator='', + ) def test_fg_export(self): self._export_test( 'samples/model_config/taobao_fg_export.config', self._extract_rtp_data, separator='', - test_multi=False) + test_multi=False, + ) def test_export_with_asset(self): pipeline_config_path = 'samples/model_config/taobao_fg.config' @@ -144,7 +151,11 @@ def _post_check_func(pipeline_config): --pipeline_config_path %s --checkpoint_path %s --export_dir %s - """ % (pipeline_config_path, ckpt_path, export_dir) + """ % ( + pipeline_config_path, + ckpt_path, + export_dir, + ) proc = test_utils.run_cmd(export_cmd, '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() @@ -155,20 +166,24 @@ def _post_check_func(pipeline_config): test_utils.test_single_train_eval( pipeline_config_path, test_dir=test_dir, - post_check_func=_post_check_func)) + post_check_func=_post_check_func, + )) def test_multi_class_predict(self): self._export_test( 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', extract_data_func=self._extract_data, - keys=['probs', 'logits', 'probs_y', 'logits_y', 'y']) - - def _export_test(self, - pipeline_config_path, - extract_data_func=None, - separator=',', - keys=['probs'], - test_multi=True): + keys=['probs', 'logits', 'probs_y', 'logits_y', 'y'], + ) + + def _export_test( + self, + pipeline_config_path, + extract_data_func=None, + separator=',', + keys=['probs'], + test_multi=True, + ): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) @@ -184,9 +199,11 @@ def _export_test(self, pipeline_config = config_util.get_configs_from_pipeline_file( config_path_single) if pipeline_config.export_config.multi_placeholder: - config_path_single, config_path_multi = config_path_multi, config_path_single - pipeline_config.export_config.multi_placeholder =\ - not pipeline_config.export_config.multi_placeholder + config_path_single, config_path_multi = ( + config_path_multi, + config_path_single, + ) + pipeline_config.export_config.multi_placeholder = not pipeline_config.export_config.multi_placeholder config_util.save_pipeline_config(pipeline_config, test_dir, 'pipeline_v2.config') @@ -197,7 +214,10 @@ def _export_test(self, python -m easy_rec.python.export --pipeline_config_path %s --export_dir %s - """ % (config_path_multi, export_dir_multi) + """ % ( + config_path_multi, + export_dir_multi, + ) proc = test_utils.run_cmd(export_cmd, '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() @@ -209,7 +229,10 @@ def _export_test(self, python -m easy_rec.python.predict --pipeline_config_path %s --output_path %s - """ % (config_path_single, result_path) + """ % ( + config_path_single, + result_path, + ) proc = test_utils.run_cmd(predict_cmd % (), '%s/log_%s.txt' % (test_dir, 'predict')) proc.wait() @@ -230,21 +253,25 @@ def _export_test(self, export_dir_single, cmp_result, keys=keys, - separator=separator) + separator=separator, + ) if test_multi: self._predict_and_check( test_data_path, export_dir_multi, cmp_result, keys=keys, - separator=separator) + separator=separator, + ) test_utils.clean_up(test_dir) - def _test_big_model_export(self, - pipeline_config_path, - test_data_path, - extract_data_func=None, - total_steps=50): + def _test_big_model_export( + self, + pipeline_config_path, + test_data_path, + extract_data_func=None, + total_steps=50, + ): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) @@ -270,8 +297,13 @@ def _test_big_model_export(self, --redis_threads 1 --redis_write_kv 1 --verbose 1 - """ % (config_path, export_dir, test_data_path, os.environ['redis_url'], - os.environ['redis_passwd']) + """ % ( + config_path, + export_dir, + test_data_path, + os.environ['redis_url'], + os.environ['redis_passwd'], + ) proc = test_utils.run_cmd(export_cmd, '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() @@ -288,7 +320,11 @@ def _test_big_model_export(self, --pipeline_config_path %s --input_path %s --output_path %s - """ % (config_path, test_data_path, result_path) + """ % ( + config_path, + test_data_path, + result_path, + ) proc = test_utils.run_cmd(predict_cmd % (), '%s/log_%s.txt' % (test_dir, 'predict')) proc.wait() @@ -307,7 +343,8 @@ def _test_big_model_export(self, @unittest.skipIf( 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd') + 'Only execute when redis is available: redis_url, redis_passwd', + ) def test_big_model_export(self): pipeline_config_path = 'samples/model_config/multi_tower_export.config' test_data_path = 'data/test/export/data.csv' @@ -318,7 +355,8 @@ def test_big_model_export(self): @unittest.skipIf( 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd') + 'Only execute when redis is available: redis_url, redis_passwd', + ) def test_big_model_deepfm_export(self): pipeline_config_path = 'samples/model_config/deepfm_combo_on_avazu_ctr.config' test_data_path = 'data/test/dwd_avazu_ctr_deepmodel_10w.csv' @@ -329,29 +367,33 @@ def test_big_model_deepfm_export(self): @unittest.skipIf( 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd') + 'Only execute when redis is available: redis_url, redis_passwd', + ) def test_big_model_din_export(self): pipeline_config_path = 'samples/model_config/din_on_taobao.config' test_data_path = 'data/test/tb_data/taobao_test_data' self._test_big_model_export( pipeline_config_path, test_data_path, - extract_data_func=functools.partial(self._extract_data, offset=2)) + extract_data_func=functools.partial(self._extract_data, offset=2), + ) @unittest.skipIf( 'redis_url' not in os.environ, - 'Only execute when redis is available: redis_url, redis_passwd') + 'Only execute when redis is available: redis_url, redis_passwd', + ) def test_big_model_wide_and_deep_export(self): pipeline_config_path = 'samples/model_config/wide_and_deep_two_opti.config' test_data_path = 'data/test/dwd_avazu_ctr_deepmodel_10w.csv' self._test_big_model_export( pipeline_config_path, test_data_path, - extract_data_func=functools.partial(self._extract_data)) + extract_data_func=functools.partial(self._extract_data), + ) @unittest.skipIf( 'redis_url' not in os.environ or '-PAI' not in tf.__version__, - 'Only execute when pai-tf and redis is available: redis_url, redis_passwd' + 'Only execute when pai-tf and redis is available: redis_url, redis_passwd', ) def test_big_model_embedding_variable_export(self): pipeline_config_path = 'samples/model_config/taobao_fg_ev.config' @@ -360,14 +402,16 @@ def test_big_model_embedding_variable_export(self): pipeline_config_path, test_data_path, self._extract_rtp_data, - total_steps=1000) + total_steps=1000, + ) @unittest.skipIf( 'oss_endpoint' not in os.environ or 'oss_ak' not in os.environ or 'oss_sk' not in os.environ or 'oss_path' not in os.environ or '-PAI' not in tf.__version__, 'Only execute oss params(oss_endpoint,oss_ak,oss_sk) are specified,' - 'and pai-tf is available.') + 'and pai-tf is available.', + ) def test_big_model_embedding_variable_oss_export(self): pipeline_config_path = 'samples/model_config/taobao_fg_ev.config' test_data_path = 'data/test/rtp/taobao_valid_feature.txt' @@ -375,14 +419,16 @@ def test_big_model_embedding_variable_oss_export(self): pipeline_config_path, test_data_path, self._extract_rtp_data, - total_steps=100) + total_steps=100, + ) @unittest.skipIf( 'oss_endpoint' not in os.environ or 'oss_ak' not in os.environ or 'oss_sk' not in os.environ or 'oss_path' not in os.environ or '-PAI' not in tf.__version__, 'Only execute oss params(oss_endpoint,oss_ak,oss_sk) are specified,' - 'and pai-tf is available.') + 'and pai-tf is available.', + ) def test_big_model_embedding_variable_v2_oss_export(self): pipeline_config_path = 'samples/model_config/taobao_fg_ev_v2.config' test_data_path = 'data/test/rtp/taobao_valid_feature.txt' @@ -390,13 +436,16 @@ def test_big_model_embedding_variable_v2_oss_export(self): pipeline_config_path, test_data_path, self._extract_rtp_data, - total_steps=100) + total_steps=100, + ) - def _test_big_model_export_to_oss(self, - pipeline_config_path, - test_data_path, - extract_data_func=None, - total_steps=50): + def _test_big_model_export_to_oss( + self, + pipeline_config_path, + test_data_path, + extract_data_func=None, + total_steps=50, + ): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) @@ -424,9 +473,15 @@ def _test_big_model_export_to_oss(self, --oss_timeout 10 --oss_write_kv 1 --verbose 1 - """ % (config_path, export_dir, test_data_path, os.environ['oss_path'], - os.environ['oss_endpoint'], os.environ['oss_ak'], - os.environ['oss_sk']) + """ % ( + config_path, + export_dir, + test_data_path, + os.environ['oss_path'], + os.environ['oss_endpoint'], + os.environ['oss_ak'], + os.environ['oss_sk'], + ) proc = test_utils.run_cmd(export_cmd, '%s/log_%s.txt' % (test_dir, 'export')) proc.wait() @@ -443,7 +498,11 @@ def _test_big_model_export_to_oss(self, --pipeline_config_path %s --input_path %s --output_path %s - """ % (config_path, test_data_path, result_path) + """ % ( + config_path, + test_data_path, + result_path, + ) proc = test_utils.run_cmd(predict_cmd, '%s/log_%s.txt' % (test_dir, 'predict')) proc.wait() @@ -462,7 +521,7 @@ def _test_big_model_export_to_oss(self, @unittest.skipIf( 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', ) def test_big_model_export_to_oss(self): pipeline_config_path = 'samples/model_config/multi_tower_export.config' @@ -474,7 +533,7 @@ def test_big_model_export_to_oss(self): @unittest.skipIf( 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', ) def test_big_model_deepfm_export_to_oss(self): pipeline_config_path = 'samples/model_config/deepfm_combo_on_avazu_ctr.config' @@ -486,7 +545,7 @@ def test_big_model_deepfm_export_to_oss(self): @unittest.skipIf( 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', ) def test_big_model_din_export_to_oss(self): pipeline_config_path = 'samples/model_config/din_on_taobao.config' @@ -494,11 +553,12 @@ def test_big_model_din_export_to_oss(self): self._test_big_model_export_to_oss( pipeline_config_path, test_data_path, - extract_data_func=functools.partial(self._extract_data, offset=2)) + extract_data_func=functools.partial(self._extract_data, offset=2), + ) @unittest.skipIf( 'oss_path' not in os.environ, - 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk' + 'Only execute when oss is available: oss_path, oss_endpoint, oss_ak, oss_sk', ) def test_big_model_wide_and_deep_export_to_oss(self): pipeline_config_path = 'samples/model_config/wide_and_deep_two_opti.config' @@ -506,7 +566,8 @@ def test_big_model_wide_and_deep_export_to_oss(self): self._test_big_model_export_to_oss( pipeline_config_path, test_data_path, - extract_data_func=functools.partial(self._extract_data)) + extract_data_func=functools.partial(self._extract_data), + ) if __name__ == '__main__': diff --git a/easy_rec/python/test/hive_input_test.py b/easy_rec/python/test/hive_input_test.py index 71aeafd4b..bce5f8cd8 100644 --- a/easy_rec/python/test/hive_input_test.py +++ b/easy_rec/python/test/hive_input_test.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Define cv_input, the base class for cv tasks.""" + import logging import os import unittest @@ -59,12 +60,13 @@ def _init_config(self): def __init__(self, methodName='HiveInputTest'): super(HiveInputTest, self).__init__(methodName=methodName) - @unittest.skipIf('hive_host' not in os.environ or - 'hive_username' not in os.environ or - 'hive_table_name' not in os.environ or - 'hive_hash_fields' not in os.environ, - """Only execute hive_config var are specified,hive_host、 - hive_username、hive_table_name、hive_hash_fields is available.""") + @unittest.skipIf( + 'hive_host' not in os.environ or 'hive_username' not in os.environ or + 'hive_table_name' not in os.environ or + 'hive_hash_fields' not in os.environ, + """Only execute hive_config var are specified,hive_host、 + hive_username、hive_table_name、hive_hash_fields is available.""", + ) def test_hive_input(self): self._init_config() data_config_str = """ @@ -242,7 +244,8 @@ def test_hive_input(self): session_config = tf.ConfigProto( gpu_options=gpu_options, allow_soft_placement=True, - log_device_placement=False) + log_device_placement=False, + ) with self.test_session(config=session_config) as sess: sess.run(init_op) feature_dict, label_dict = sess.run([features, labels]) @@ -253,12 +256,13 @@ def test_hive_input(self): print(key, label_dict[key][:5]) return 0 - @unittest.skipIf('hive_host' not in os.environ or - 'hive_username' not in os.environ or - 'hive_table_name' not in os.environ or - 'hive_hash_fields' not in os.environ, - """Only execute hive_config var are specified,hive_host、 - hive_username、hive_table_name、hive_hash_fields is available.""") + @unittest.skipIf( + 'hive_host' not in os.environ or 'hive_username' not in os.environ or + 'hive_table_name' not in os.environ or + 'hive_hash_fields' not in os.environ, + """Only execute hive_config var are specified,hive_host、 + hive_username、hive_table_name、hive_hash_fields is available.""", + ) def test_mmoe(self): pipeline_config_path = 'samples/emr_script/mmoe/mmoe_census_income.config' gpus = test_utils.get_available_gpus() @@ -286,7 +290,9 @@ def test_mmoe(self): test_pipeline_config_path = os.path.join(self._test_dir, 'pipeline.config') hyperparam_str = '' train_cmd = 'python -m easy_rec.python.train_eval --pipeline_config_path %s %s' % ( - test_pipeline_config_path, hyperparam_str) + test_pipeline_config_path, + hyperparam_str, + ) proc = test_utils.run_cmd(train_cmd, '%s/log_%s.txt' % (self._test_dir, 'master')) proc.wait() diff --git a/easy_rec/python/test/hpo_test.py b/easy_rec/python/test/hpo_test.py index a570d87d0..d6ed01fcf 100644 --- a/easy_rec/python/test/hpo_test.py +++ b/easy_rec/python/test/hpo_test.py @@ -22,6 +22,7 @@ GPUOptions = config_pb2.GPUOptions else: from tensorflow.python.platform import gfile + GPUOptions = tf.GPUOptions ConfigProto = tf.ConfigProto @@ -116,7 +117,9 @@ def test_edit_config_v6(self): for i, tmp_fea in enumerate(tmp_config.feature_configs): if tmp_fea.input_names[0] >= 'site': assert tmp_fea.embedding_dim == 32, 'input_name = %s %d' % ( - tmp_fea.input_names[0], tmp_fea.embedding_dim) + tmp_fea.input_names[0], + tmp_fea.embedding_dim, + ) else: assert tmp_fea.embedding_dim == 16 @@ -165,8 +168,7 @@ def test_edit_config_v9(self): tmp_config = config_util.get_configs_from_pipeline_file(tmp_file) tmp_file = 'samples/hpo/hpo_param_v9.json' tmp_config = config_util.edit_config(tmp_config, self.load_config(tmp_file)) - assert tmp_config.train_config.fine_tune_checkpoint == \ - 'oss://easy-rec/test/experiment/ctr_v93/model.ckpt-1000' + assert tmp_config.train_config.fine_tune_checkpoint == 'oss://easy-rec/test/experiment/ctr_v93/model.ckpt-1000' def test_edit_config_v10(self): tmp_file = 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config' diff --git a/easy_rec/python/test/kafka_test.py b/easy_rec/python/test/kafka_test.py index f0da2d5d5..aa94fbff0 100644 --- a/easy_rec/python/test/kafka_test.py +++ b/easy_rec/python/test/kafka_test.py @@ -22,7 +22,7 @@ try: import kafka - from kafka import KafkaProducer, KafkaAdminClient + from kafka import KafkaAdminClient, KafkaProducer from kafka.admin import NewTopic except ImportError: logging.warning('kafka-python is not installed: %s' % traceback.format_exc()) @@ -59,8 +59,10 @@ def setUp(self): fout.write('dataDir=%s/zookeeper\n' % self._test_dir) else: fout.write(line_str) - cmd = 'bash %s/bin/zookeeper-server-start.sh %s' % (kafka_install_dir, - zookeeper_config) + cmd = 'bash %s/bin/zookeeper-server-start.sh %s' % ( + kafka_install_dir, + zookeeper_config, + ) log_file = os.path.join(self._log_dir, 'zookeeper.log') self._zookeeper_proc = test_utils.run_cmd(cmd, log_file) @@ -73,8 +75,10 @@ def setUp(self): fout.write('log.dirs=%s/kafka\n' % self._test_dir) else: fout.write(line_str) - cmd = 'bash %s/bin/kafka-server-start.sh %s' % (kafka_install_dir, - kafka_config) + cmd = 'bash %s/bin/kafka-server-start.sh %s' % ( + kafka_install_dir, + kafka_config, + ) log_file = os.path.join(self._log_dir, 'kafka_server.log') self._kafka_server_proc = test_utils.run_cmd(cmd, log_file) @@ -109,7 +113,8 @@ def _create_topic(self, num_partitions=2): NewTopic( name=self._test_topic, num_partitions=num_partitions, - replication_factor=1) + replication_factor=1, + ) ] admin_clt.create_topics(new_topics=topic_list, validate_only=False) @@ -172,7 +177,8 @@ def _generate(): # control the maximal read of each partition config_global=['max.partition.fetch.bytes=1048576'], message_key=True, - message_offset=True) + message_offset=True, + ) batch_dataset = k.batch(5) @@ -250,7 +256,8 @@ def test_kafka_train_chief_redundant(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/deepfm_combo_avazu_kafka_chief_redundant.config', self._test_dir, - num_evaluator=1) + num_evaluator=1, + ) self.assertTrue(self._success) except Exception as ex: self._success = False @@ -267,7 +274,8 @@ def test_kafka_train_v2(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_avazu_kafka_time_offset.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) except Exception as ex: @@ -277,7 +285,9 @@ def test_kafka_train_v2(self): @unittest.skipIf( 'kafka_install_dir' not in os.environ or 'oss_path' not in os.environ or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or - 'oss_sk' not in os.environ, 'Only execute when kafka is available') + 'oss_sk' not in os.environ, + 'Only execute when kafka is available', + ) def test_kafka_processor(self): self._test_kafka_processor( 'samples/model_config/taobao_fg_incr_save.config') @@ -285,7 +295,9 @@ def test_kafka_processor(self): @unittest.skipIf( 'kafka_install_dir' not in os.environ or 'oss_path' not in os.environ or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or - 'oss_sk' not in os.environ, 'Only execute when kafka is available') + 'oss_sk' not in os.environ, + 'Only execute when kafka is available', + ) def test_kafka_processor_ev(self): self._test_kafka_processor( 'samples/model_config/taobao_fg_incr_save_ev.config') @@ -300,9 +312,15 @@ def _test_kafka_processor(self, config_path): --export_dir %s/export/sep/ --oss_path=%s --oss_ak=%s --oss_sk=%s --oss_endpoint=%s --asset_files ./samples/rtp_fg/fg.json --checkpoint_path %s/train/model.ckpt-0 - """ % (self._test_dir, self._test_dir, os.environ['oss_path'], - os.environ['oss_ak'], os.environ['oss_sk'], - os.environ['oss_endpoint'], self._test_dir) + """ % ( + self._test_dir, + self._test_dir, + os.environ['oss_path'], + os.environ['oss_ak'], + os.environ['oss_sk'], + os.environ['oss_endpoint'], + self._test_dir, + ) proc = test_utils.run_cmd(export_cmd, '%s/log_export_sep.txt' % self._test_dir) proc.wait() @@ -314,7 +332,11 @@ def _test_kafka_processor(self, config_path): python -m easy_rec.python.inference.processor.test --saved_model_dir %s --input_path data/test/rtp/taobao_test_feature.txt --output_path %s/processor.out --test_dir %s - """ % (export_sep_dir, self._test_dir, self._test_dir) + """ % ( + export_sep_dir, + self._test_dir, + self._test_dir, + ) envs = dict(os.environ) envs['PROCESSOR_TEST'] = '1' proc = test_utils.run_cmd( @@ -361,7 +383,8 @@ def test_kafka_train_v3(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_avazu_kafka_time_offset2.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) except Exception as ex: diff --git a/easy_rec/python/test/local_incr_test.py b/easy_rec/python/test/local_incr_test.py index ad2d657f3..5b9e33bd9 100644 --- a/easy_rec/python/test/local_incr_test.py +++ b/easy_rec/python/test/local_incr_test.py @@ -30,7 +30,9 @@ def setUp(self): @unittest.skipIf( 'oss_path' not in os.environ or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or - 'oss_sk' not in os.environ, 'Only execute when kafka is available') + 'oss_sk' not in os.environ, + 'Only execute when kafka is available', + ) def test_incr_save(self): self._test_incr_save( 'samples/model_config/taobao_fg_incr_save_local.config') @@ -38,7 +40,9 @@ def test_incr_save(self): @unittest.skipIf( 'oss_path' not in os.environ or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or - 'oss_sk' not in os.environ, 'Only execute when kafka is available') + 'oss_sk' not in os.environ, + 'Only execute when kafka is available', + ) def test_incr_save_ev(self): self._test_incr_save( 'samples/model_config/taobao_fg_incr_save_ev_local.config') @@ -46,7 +50,9 @@ def test_incr_save_ev(self): @unittest.skipIf( 'oss_path' not in os.environ or 'oss_endpoint' not in os.environ and 'oss_ak' not in os.environ or - 'oss_sk' not in os.environ, 'Only execute when kafka is available') + 'oss_sk' not in os.environ, + 'Only execute when kafka is available', + ) def test_incr_save_share_ev(self): self._test_incr_save( 'samples/model_config/taobao_fg_incr_save_share_ev_local.config') @@ -60,16 +66,23 @@ def _test_incr_save(self, config_path): edit_config_json={ 'train_config.incr_save_config.fs.mount_path': os.path.join(self._test_dir, 'train/incr_save/') - }) + }, + ) self.assertTrue(success) export_cmd = """ python -m easy_rec.python.export --pipeline_config_path %s/pipeline.config --export_dir %s/export/sep/ --oss_path=%s --oss_ak=%s --oss_sk=%s --oss_endpoint=%s --asset_files ./samples/rtp_fg/fg.json --checkpoint_path %s/train/model.ckpt-0 - """ % (self._test_dir, self._test_dir, os.environ['oss_path'], - os.environ['oss_ak'], os.environ['oss_sk'], - os.environ['oss_endpoint'], self._test_dir) + """ % ( + self._test_dir, + self._test_dir, + os.environ['oss_path'], + os.environ['oss_ak'], + os.environ['oss_sk'], + os.environ['oss_endpoint'], + self._test_dir, + ) proc = test_utils.run_cmd(export_cmd, '%s/log_export_sep.txt' % self._test_dir) proc.wait() @@ -81,7 +94,11 @@ def _test_incr_save(self, config_path): python -m easy_rec.python.inference.processor.test --saved_model_dir %s --input_path data/test/rtp/taobao_test_feature.txt --output_path %s/processor.out --test_dir %s - """ % (export_sep_dir, self._test_dir, self._test_dir) + """ % ( + export_sep_dir, + self._test_dir, + self._test_dir, + ) envs = dict(os.environ) envs['PROCESSOR_TEST'] = '1' proc = test_utils.run_cmd( diff --git a/easy_rec/python/test/loss_test.py b/easy_rec/python/test/loss_test.py index f78b74ce6..bf5d9a265 100644 --- a/easy_rec/python/test/loss_test.py +++ b/easy_rec/python/test/loss_test.py @@ -1,12 +1,13 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import tensorflow as tf -from easy_rec.python.loss.circle_loss import circle_loss -from easy_rec.python.loss.circle_loss import get_anchor_positive_triplet_mask - -from easy_rec.python.loss.f1_reweight_loss import f1_reweight_sigmoid_cross_entropy # NOQA - -from easy_rec.python.loss.softmax_loss_with_negative_mining import softmax_loss_with_negative_mining # NOQA +from easy_rec.python.loss.circle_loss import ( # NOQA + circle_loss, get_anchor_positive_triplet_mask, +) +from easy_rec.python.loss.f1_reweight_loss import ( # NOQA + f1_reweight_sigmoid_cross_entropy,) +from easy_rec.python.loss.softmax_loss_with_negative_mining import ( # NOQA + softmax_loss_with_negative_mining,) if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -26,12 +27,22 @@ def test_f1_reweighted_loss(self): def test_softmax_loss_with_negative_mining(self): print('test_softmax_loss_with_negative_mining') - user_emb = tf.constant([[0.1, 0.5, 0.3], [0.8, -0.1, 0.3], [0.28, 0.3, 0.9], - [0.37, 0.45, 0.93], [-0.7, 0.15, 0.03], - [0.18, 0.9, -0.3]]) - item_emb = tf.constant([[0.1, -0.5, 0.3], [0.8, -0.31, 0.3], - [0.7, -0.45, 0.15], [0.08, -0.31, -0.9], - [-0.7, 0.85, 0.03], [0.18, 0.89, -0.3]]) + user_emb = tf.constant([ + [0.1, 0.5, 0.3], + [0.8, -0.1, 0.3], + [0.28, 0.3, 0.9], + [0.37, 0.45, 0.93], + [-0.7, 0.15, 0.03], + [0.18, 0.9, -0.3], + ]) + item_emb = tf.constant([ + [0.1, -0.5, 0.3], + [0.8, -0.31, 0.3], + [0.7, -0.45, 0.15], + [0.08, -0.31, -0.9], + [-0.7, 0.85, 0.03], + [0.18, 0.89, -0.3], + ]) label = tf.constant([1, 1, 0, 0, 1, 1]) loss = softmax_loss_with_negative_mining( @@ -42,10 +53,17 @@ def test_softmax_loss_with_negative_mining(self): def test_circle_loss(self): print('test_circle_loss') - emb = tf.constant([[0.1, 0.2, 0.15, 0.1], [0.3, 0.6, 0.45, 0.3], - [0.13, 0.6, 0.45, 0.3], [0.3, 0.26, 0.45, 0.3], - [0.3, 0.6, 0.5, 0.13], [0.08, 0.43, 0.21, 0.6]], - dtype=tf.float32) + emb = tf.constant( + [ + [0.1, 0.2, 0.15, 0.1], + [0.3, 0.6, 0.45, 0.3], + [0.13, 0.6, 0.45, 0.3], + [0.3, 0.26, 0.45, 0.3], + [0.3, 0.6, 0.5, 0.13], + [0.08, 0.43, 0.21, 0.6], + ], + dtype=tf.float32, + ) label = tf.constant([1, 1, 2, 2, 3, 3]) loss = circle_loss(emb, label, label, margin=0.25, gamma=64) with self.test_session() as sess: @@ -56,17 +74,31 @@ def test_triplet_mask(self): print('test_triplet_mask') label = tf.constant([1, 1, 2, 2, 3, 3, 4, 5]) positive_mask = tf.constant( - [[0., 1., 0., 0., 0., 0., 0., 0.], [1., 0., 0., 0., 0., 0., 0., 0.], - [0., 0., 0., 1., 0., 0., 0., 0.], [0., 0., 1., 0., 0., 0., 0., 0.], - [0., 0., 0., 0., 0., 1., 0., 0.], [0., 0., 0., 0., 1., 0., 0., 0.], - [0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0., 0.]], - dtype=tf.float32) + [ + [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + dtype=tf.float32, + ) negative_mask = tf.constant( - [[0., 0., 1., 1., 1., 1., 1., 1.], [0., 0., 1., 1., 1., 1., 1., 1.], - [1., 1., 0., 0., 1., 1., 1., 1.], [1., 1., 0., 0., 1., 1., 1., 1.], - [1., 1., 1., 1., 0., 0., 1., 1.], [1., 1., 1., 1., 0., 0., 1., 1.], - [1., 1., 1., 1., 1., 1., 0., 1.], [1., 1., 1., 1., 1., 1., 1., 0.]], - dtype=tf.float32) + [ + [0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0], + [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0], + ], + dtype=tf.float32, + ) with self.test_session(): pos_mask = get_anchor_positive_triplet_mask(label, label) self.assertAllEqual(positive_mask, pos_mask) diff --git a/easy_rec/python/test/odps_command.py b/easy_rec/python/test/odps_command.py index 33298f727..0943887a7 100644 --- a/easy_rec/python/test/odps_command.py +++ b/easy_rec/python/test/odps_command.py @@ -16,10 +16,12 @@ def __init__(self, odps_oss_config): Args: odps_oss_config: instance of easy_rec.python.utils.odps_test_util.OdpsOSSConfig """ - self.bucket = get_oss_bucket(odps_oss_config.oss_key, - odps_oss_config.oss_secret, - odps_oss_config.endpoint, - odps_oss_config.bucket_name) + self.bucket = get_oss_bucket( + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, + ) self.bucket_name = odps_oss_config.bucket_name self.temp_dir = odps_oss_config.temp_dir self.log_path = odps_oss_config.log_dir @@ -42,15 +44,22 @@ def run_odps_cmd(self, script_file): log_file = os.path.join(self.log_path, file_name) if self.odps_config_path is None: - cmd = 'nohup %s -f %s > %s.log 2>&1' % (self.odpscmd, exec_file_path, - log_file) + cmd = 'nohup %s -f %s > %s.log 2>&1' % ( + self.odpscmd, + exec_file_path, + log_file, + ) else: cmd = 'nohup %s --config=%s -f %s > %s.log 2>&1' % ( - self.odpscmd, self.odps_config_path, exec_file_path, log_file) + self.odpscmd, + self.odps_config_path, + exec_file_path, + log_file, + ) logging.info('will run cmd: %s' % (cmd)) proc = subprocess.Popen(cmd, shell=True) proc.wait() - if (proc.returncode == 0): + if proc.returncode == 0: logging.info('%s run succeed' % script_file) else: raise ValueError('%s run FAILED: please check log file:%s.log' % diff --git a/easy_rec/python/test/odps_local_run.py b/easy_rec/python/test/odps_local_run.py index 8c1d15274..a6cc4d121 100644 --- a/easy_rec/python/test/odps_local_run.py +++ b/easy_rec/python/test/odps_local_run.py @@ -11,11 +11,12 @@ from easy_rec.python.test.odps_command import OdpsCommand from easy_rec.python.test.odps_test_prepare import prepare -from easy_rec.python.test.odps_test_util import OdpsOSSConfig -from easy_rec.python.test.odps_test_util import delete_oss_path -from easy_rec.python.test.odps_test_util import get_oss_bucket from easy_rec.python.utils import test_utils +from easy_rec.python.test.odps_test_util import ( # NOQA + OdpsOSSConfig, delete_oss_path, get_oss_bucket, +) + logging.basicConfig( level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') @@ -37,7 +38,7 @@ def tearDown(self): def test_deepfm_local_with_common_io(self): start = [ 'deep_fm/create_external_deepfm_table.sql', - 'deep_fm/create_inner_deepfm_table.sql' + 'deep_fm/create_inner_deepfm_table.sql', ] end = ['deep_fm/drop_table.sql'] odps_cmd = OdpsCommand(odps_oss_config) @@ -79,8 +80,12 @@ def test_deepfm_local_with_common_io(self): prepare(odps_oss_config) tf.test.main() # delete oss path - bucket = get_oss_bucket(odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name) + bucket = get_oss_bucket( + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, + ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) # delete tmp shutil.rmtree(odps_oss_config.temp_dir) diff --git a/easy_rec/python/test/odps_run.py b/easy_rec/python/test/odps_run.py index 84bd44f9b..8943ab509 100644 --- a/easy_rec/python/test/odps_run.py +++ b/easy_rec/python/test/odps_run.py @@ -12,11 +12,12 @@ from easy_rec.python.test.odps_test_cls import OdpsTest from easy_rec.python.test.odps_test_prepare import prepare -from easy_rec.python.test.odps_test_util import OdpsOSSConfig -from easy_rec.python.test.odps_test_util import delete_oss_path -from easy_rec.python.test.odps_test_util import get_oss_bucket from easy_rec.python.utils import config_util +from easy_rec.python.test.odps_test_util import ( # NOQA + OdpsOSSConfig, delete_oss_path, get_oss_bucket, +) + logging.basicConfig( level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') @@ -24,14 +25,16 @@ class TestPipelineOnOdps(tf.test.TestCase): - """train eval export test on odps.""" + """Train eval export test on odps.""" def test_deepfm(self): start_files = ['deep_fm/create_inner_deepfm_table.sql'] test_files = [ - 'deep_fm/train_deepfm_model.sql', 'deep_fm/eval_deepfm.sql', - 'deep_fm/export_deepfm.sql', 'deep_fm/predict_deepfm.sql', - 'deep_fm/export_rtp_ckpt.sql' + 'deep_fm/train_deepfm_model.sql', + 'deep_fm/eval_deepfm.sql', + 'deep_fm/export_deepfm.sql', + 'deep_fm/predict_deepfm.sql', + 'deep_fm/export_rtp_ckpt.sql', ] end_file = ['deep_fm/drop_table.sql'] @@ -113,7 +116,8 @@ def test_best_exporter(self): tot.start_test() config_path = os.path.join( odps_oss_config.temp_dir, - 'configs/dwd_avazu_ctr_deepmodel_ext_best_export.config') + 'configs/dwd_avazu_ctr_deepmodel_ext_best_export.config', + ) config = config_util.get_configs_from_pipeline_file(config_path) model_dir = config.model_dir logging.info('raw model_dir = %s' % model_dir) @@ -122,9 +126,12 @@ def test_best_exporter(self): model_dir = model_dir[spos:] logging.info('stripped model_dir = %s' % model_dir) - bucket = get_oss_bucket(odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, - odps_oss_config.bucket_name) + bucket = get_oss_bucket( + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, + ) best_ckpt_prefix = os.path.join(model_dir, 'best_ckpt/model.ckpt') best_ckpts = [ x.key @@ -149,8 +156,9 @@ def test_embedding_variable(self): 'embedding_variable/create_table.sql', ] test_files = [ - 'embedding_variable/train.sql', 'embedding_variable/train_work_que.sql', - 'embedding_variable/export.sql' + 'embedding_variable/train.sql', + 'embedding_variable/train_work_que.sql', + 'embedding_variable/export.sql', ] end_file = ['embedding_variable/drop_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) @@ -172,7 +180,8 @@ def test_boundary_test(self): test_files = [ 'boundary/train_multi_tower_model.sql', 'boundary/finetune_multi_tower_model.sql', - 'boundary/finetune_multi_tower_conti.sql', 'boundary/train_compat.sql' + 'boundary/finetune_multi_tower_conti.sql', + 'boundary/train_compat.sql', ] end_file = ['boundary/drop_table.sql'] tot = OdpsTest(start_files, test_files, end_file, odps_oss_config) @@ -217,7 +226,7 @@ def test_vector_retrieve(self): '--is_outer', type=int, default=1, - help='is outer pai or inner pai, the arguments are differed slightly due to history reasons' + help='is outer pai or inner pai, the arguments are differed slightly due to history reasons', ) args, unknown_args = parser.parse_known_args() sys.argv = [sys.argv[0]] @@ -248,7 +257,11 @@ def test_vector_retrieve(self): prepare(odps_oss_config) tf.test.main() - bucket = get_oss_bucket(odps_oss_config.oss_key, odps_oss_config.oss_secret, - odps_oss_config.endpoint, odps_oss_config.bucket_name) + bucket = get_oss_bucket( + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, + ) delete_oss_path(bucket, odps_oss_config.exp_dir, odps_oss_config.bucket_name) shutil.rmtree(odps_oss_config.temp_dir) diff --git a/easy_rec/python/test/odps_test_prepare.py b/easy_rec/python/test/odps_test_prepare.py index e4b0c23d2..d54017acd 100644 --- a/easy_rec/python/test/odps_test_prepare.py +++ b/easy_rec/python/test/odps_test_prepare.py @@ -107,7 +107,10 @@ def change_files(odps_oss_config, file_path): # tmp_e = tmp_e.replace('.aliyuncs.com', '.oss-internal.aliyun-inc.com') if '-Dbuckets=' in line: line = '-Dbuckets=oss://%s/?role_arn=%s&host=%s\n' % ( - odps_oss_config.bucket_name, odps_oss_config.arn, tmp_e) + odps_oss_config.bucket_name, + odps_oss_config.arn, + tmp_e, + ) elif '-Darn=' in line or '-DossHost' in line: continue line = line.replace('{OSS_BUCKET_NAME}', odps_oss_config.bucket_name) @@ -136,10 +139,12 @@ def put_data_to_bucket(odps_oss_config): Args: odps_oss_config: odps oss config obj """ - test_bucket = get_oss_bucket(odps_oss_config.oss_key, - odps_oss_config.oss_secret, - odps_oss_config.endpoint, - odps_oss_config.bucket_name) + test_bucket = get_oss_bucket( + odps_oss_config.oss_key, + odps_oss_config.oss_secret, + odps_oss_config.endpoint, + odps_oss_config.bucket_name, + ) for sub_dir in ['configs']: for root, dirs, files in os.walk( os.path.join(odps_oss_config.temp_dir, sub_dir)): @@ -154,10 +159,12 @@ def put_data_to_bucket(odps_oss_config): def prepare(odps_oss_config): logging.info('temp_dir = %s' % odps_oss_config.temp_dir) - ali_bucket = get_oss_bucket(odps_oss_config.ali_oss_key, - odps_oss_config.ali_oss_secret, - odps_oss_config.ali_bucket_endpoint, - odps_oss_config.ali_bucket_name) + ali_bucket = get_oss_bucket( + odps_oss_config.ali_oss_key, + odps_oss_config.ali_oss_secret, + odps_oss_config.ali_bucket_endpoint, + odps_oss_config.ali_bucket_name, + ) shutil.copytree(odps_oss_config.script_path, odps_oss_config.temp_dir) logging.info('start down data') download_data(ali_bucket, odps_oss_config.temp_dir) diff --git a/easy_rec/python/test/odps_test_util.py b/easy_rec/python/test/odps_test_util.py index 35dc7f743..2cdcd9684 100644 --- a/easy_rec/python/test/odps_test_util.py +++ b/easy_rec/python/test/odps_test_util.py @@ -10,16 +10,15 @@ try: from datahub import DataHub - from datahub.exceptions import InvalidOperationException - from datahub.exceptions import ResourceExistException + from datahub.exceptions import ( # NOQA + InvalidOperationException, ResourceExistException, + ) + # from datahub.exceptions import LimitExceededException # from datahub.exceptions import ResourceNotFoundException # from datahub.models import BlobRecord # from datahub.models import CursorType - from datahub.models import FieldType - from datahub.models import RecordSchema - from datahub.models import RecordType - from datahub.models import TupleRecord + from datahub.models import FieldType, RecordSchema, RecordType, TupleRecord except Exception: logging.error( 'DataHub is not installed, please installed it by: pip install pydatahub') @@ -145,7 +144,7 @@ def get_input_type(self, input_type): 'STRING': FieldType.STRING, 'BOOLEAN': FieldType.BOOLEAN, 'FLOAT32': FieldType.DOUBLE, - 'FLOAT64': FieldType.DOUBLE + 'FLOAT64': FieldType.DOUBLE, } return DhDict.get(input_type) @@ -175,7 +174,8 @@ def init_dh_and_odps(self): 7, 3, record_schema, - comment='EasyRecTest') + comment='EasyRecTest', + ) logging.info('create tuple topic %s success!' % self.dh_topic) except ResourceExistException: logging.info('topic %s already exist!' % self.dh_topic) diff --git a/easy_rec/python/test/pre_check_test.py b/easy_rec/python/test/pre_check_test.py index 58b295157..d62befae6 100644 --- a/easy_rec/python/test/pre_check_test.py +++ b/easy_rec/python/test/pre_check_test.py @@ -29,7 +29,8 @@ def test_csv_input_train_with_check(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_taobao.config', self._test_dir, - check_mode=True) + check_mode=True, + ) self.assertTrue(self._success) def test_rtp_input_train_with_check(self): diff --git a/easy_rec/python/test/predictor_test.py b/easy_rec/python/test/predictor_test.py index 7ad8ae36e..0f58432e9 100644 --- a/easy_rec/python/test/predictor_test.py +++ b/easy_rec/python/test/predictor_test.py @@ -56,10 +56,24 @@ def test_lookup_pred(self): def test_pred_dict(self): predictor = Predictor('data/test/inference/tb_multitower_export/') field_keys = [ - 'pid', 'adgroup_id', 'cate_id', 'campaign_id', 'customer', 'brand', - 'user_id', 'cms_segid', 'cms_group_id', 'final_gender_code', - 'age_level', 'pvalue_level', 'shopping_level', 'occupation', - 'new_user_class_level', 'tag_category_list', 'tag_brand_list', 'price' + 'pid', + 'adgroup_id', + 'cate_id', + 'campaign_id', + 'customer', + 'brand', + 'user_id', + 'cms_segid', + 'cms_group_id', + 'final_gender_code', + 'age_level', + 'pvalue_level', + 'shopping_level', + 'occupation', + 'new_user_class_level', + 'tag_category_list', + 'tag_brand_list', + 'price', ] with open(self._test_path, 'r') as fin: reader = csv.reader(fin) @@ -74,10 +88,24 @@ def test_pred_placeholder_named_by_input(self): predictor = Predictor( 'data/test/inference/tb_multitower_placeholder_rename_export/') field_keys = [ - 'pid', 'adgroup_id', 'cate_id', 'campaign_id', 'customer', 'brand', - 'user_id', 'cms_segid', 'cms_group_id', 'final_gender_code', - 'age_level', 'pvalue_level', 'shopping_level', 'occupation', - 'new_user_class_level', 'tag_category_list', 'tag_brand_list', 'price' + 'pid', + 'adgroup_id', + 'cate_id', + 'campaign_id', + 'customer', + 'brand', + 'user_id', + 'cms_segid', + 'cms_group_id', + 'final_gender_code', + 'age_level', + 'pvalue_level', + 'shopping_level', + 'occupation', + 'new_user_class_level', + 'tag_category_list', + 'tag_brand_list', + 'price', ] with open(self._test_path, 'r') as fin: reader = csv.reader(fin) @@ -108,10 +136,24 @@ def test_fm_pred_list(self): def test_fm_pred_dict(self): predictor = Predictor('data/test/inference/fm_export/') field_keys = [ - 'pid', 'adgroup_id', 'cate_id', 'campaign_id', 'customer', 'brand', - 'user_id', 'cms_segid', 'cms_group_id', 'final_gender_code', - 'age_level', 'pvalue_level', 'shopping_level', 'occupation', - 'new_user_class_level', 'tag_category_list', 'tag_brand_list', 'price' + 'pid', + 'adgroup_id', + 'cate_id', + 'campaign_id', + 'customer', + 'brand', + 'user_id', + 'cms_segid', + 'cms_group_id', + 'final_gender_code', + 'age_level', + 'pvalue_level', + 'shopping_level', + 'occupation', + 'new_user_class_level', + 'tag_category_list', + 'tag_brand_list', + 'price', ] with open(self._test_path, 'r') as fin: reader = csv.reader(fin) @@ -125,7 +167,6 @@ def test_fm_pred_dict(self): class PredictorTestOnDS(tf.test.TestCase): def setUp(self): - self._test_dir = test_utils.get_tmp_dir() self._test_output_path = None logging.info('Testing %s.%s' % (type(self).__name__, self._testMethodName)) @@ -148,7 +189,8 @@ def test_local_pred(self): saved_model_dir, pipeline_config.data_config, output_sep=';', - selected_cols='') + selected_cols='', + ) predictor.predict_impl( test_input_path, @@ -156,10 +198,13 @@ def test_local_pred(self): reserved_cols='ALL_COLUMNS', output_cols='ALL_COLUMNS', slice_id=0, - slice_num=1) - header_truth = 'logits;probs;clk;buy;pid;adgroup_id;cate_id;campaign_id;customer;'\ - 'brand;user_id;cms_segid;cms_group_id;final_gender_code;age_level;pvalue_level;' \ - 'shopping_level;occupation;new_user_class_level;tag_category_list;tag_brand_list;price' + slice_num=1, + ) + header_truth = ( + 'logits;probs;clk;buy;pid;adgroup_id;cate_id;campaign_id;customer;' + 'brand;user_id;cms_segid;cms_group_id;final_gender_code;age_level;pvalue_level;' + 'shopping_level;occupation;new_user_class_level;tag_category_list;tag_brand_list;price' + ) with open(self._test_output_path + '/part-0.csv', 'r') as f: output_res = f.readlines() @@ -182,7 +227,8 @@ def test_local_pred_with_header(self): pipeline_config.data_config, with_header=True, output_sep=';', - selected_cols='') + selected_cols='', + ) predictor.predict_impl( test_input_path, @@ -190,10 +236,13 @@ def test_local_pred_with_header(self): reserved_cols='ALL_COLUMNS', output_cols='ALL_COLUMNS', slice_id=0, - slice_num=1) - header_truth = 'logits;probs;clk;buy;pid;adgroup_id;cate_id;campaign_id;customer;'\ - 'brand;user_id;cms_segid;cms_group_id;final_gender_code;age_level;pvalue_level;' \ - 'shopping_level;occupation;new_user_class_level;tag_category_list;tag_brand_list;price' + slice_num=1, + ) + header_truth = ( + 'logits;probs;clk;buy;pid;adgroup_id;cate_id;campaign_id;customer;' + 'brand;user_id;cms_segid;cms_group_id;final_gender_code;age_level;pvalue_level;' + 'shopping_level;occupation;new_user_class_level;tag_category_list;tag_brand_list;price' + ) with open(self._test_output_path + '/part-0.csv', 'r') as f: output_res = f.readlines() @@ -228,7 +277,8 @@ def test_local_pred_with_part_col(self): saved_model_dir, pipeline_config.data_config, output_sep=';', - selected_cols='') + selected_cols='', + ) predictor.predict_impl( test_input_path, @@ -236,7 +286,8 @@ def test_local_pred_with_part_col(self): reserved_cols='clk,buy,user_id,adgroup_id', output_cols='probs', slice_id=0, - slice_num=1) + slice_num=1, + ) header_truth = 'probs;clk;buy;user_id;adgroup_id' with open(self._test_output_path + '/part-0.csv', 'r') as f: @@ -259,14 +310,16 @@ def test_local_pred_rtp(self): saved_model_dir, pipeline_config.data_config, output_sep=';', - selected_cols='0,3') + selected_cols='0,3', + ) predictor.predict_impl( test_input_path, self._test_output_path, reserved_cols='ALL_COLUMNS', output_cols='ALL_COLUMNS', slice_id=0, - slice_num=1) + slice_num=1, + ) header_truth = 'logits;probs;clk;no_used_1;no_used_2;features' with open(self._test_output_path + '/part-0.csv', 'r') as f: output_res = f.readlines() @@ -288,14 +341,16 @@ def test_local_pred_rtp_with_part_col(self): saved_model_dir, pipeline_config.data_config, output_sep=';', - selected_cols='0,3') + selected_cols='0,3', + ) predictor.predict_impl( test_input_path, self._test_output_path, reserved_cols='clk,features,no_used_1', output_cols='ALL_COLUMNS', slice_id=0, - slice_num=1) + slice_num=1, + ) header_truth = 'logits;probs;clk;features;no_used_1' with open(self._test_output_path + '/part-0.csv', 'r') as f: output_res = f.readlines() @@ -316,7 +371,8 @@ def test_local_pred_embedding(self): pipeline_config.data_config, ds_vector_recall=True, output_sep=';', - selected_cols='pid,adgroup_id,cate_id,campaign_id,customer,brand,price') + selected_cols='pid,adgroup_id,cate_id,campaign_id,customer,brand,price', + ) predictor.predict_impl( test_input_path, @@ -324,7 +380,8 @@ def test_local_pred_embedding(self): reserved_cols='adgroup_id', output_cols='item_emb', slice_id=0, - slice_num=1) + slice_num=1, + ) with open(self._test_output_path + '/part-0.csv', 'r') as f: output_res = f.readlines() diff --git a/easy_rec/python/test/rtp_convert_test.py b/easy_rec/python/test/rtp_convert_test.py index 4210c9dc2..8f7aad70e 100644 --- a/easy_rec/python/test/rtp_convert_test.py +++ b/easy_rec/python/test/rtp_convert_test.py @@ -22,7 +22,7 @@ def test_rtp_convert(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'fg_multi_tower.config') - convert_cmd = """ + convert_cmd = (""" python -m easy_rec.python.tools.convert_rtp_fg --rtp_fg samples/rtp_fg/fg.json --label clk @@ -32,7 +32,7 @@ def test_rtp_convert(self): --train_input_path data/test/rtp/taobao_train_feature.txt --eval_input_path data/test/rtp/taobao_test_feature.txt --selected_cols 0,3 --num_steps 400 - """ % pipeline_config_path + """ % pipeline_config_path) proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() @@ -46,7 +46,7 @@ def test_rtp_convert_bucketize(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'fg_multi_tower.config') - convert_cmd = """ + convert_cmd = (""" python -m easy_rec.python.tools.convert_rtp_fg --rtp_fg samples/rtp_fg/fg_bucketize.json --label clk @@ -56,7 +56,7 @@ def test_rtp_convert_bucketize(self): --train_input_path data/test/rtp/taobao_train_bucketize_feature.txt --eval_input_path data/test/rtp/taobao_test_bucketize_feature.txt --selected_cols 0,3 --num_steps 400 - """ % pipeline_config_path + """ % pipeline_config_path) proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() @@ -70,7 +70,7 @@ def test_rtp_convert_bucketize_v2(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'fg_multi_tower.config') - convert_cmd = """ + convert_cmd = (""" python -m easy_rec.python.tools.convert_rtp_fg --rtp_fg samples/rtp_fg/fg_bucketize_v2.json --label clk @@ -80,7 +80,7 @@ def test_rtp_convert_bucketize_v2(self): --train_input_path data/test/rtp/taobao_train_feature.txt --eval_input_path data/test/rtp/taobao_test_feature.txt --selected_cols 0,3 --num_steps 400 - """ % pipeline_config_path + """ % pipeline_config_path) proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() @@ -101,7 +101,7 @@ def test_rtp_convert_test_model_config(self): test_dir = test_utils.get_tmp_dir() logging.info('test dir: %s' % test_dir) pipeline_config_path = os.path.join(test_dir, 'fg_wide_and_deep.config') - convert_cmd = """ + convert_cmd = (""" python -m easy_rec.python.tools.convert_rtp_fg --rtp_fg samples/rtp_fg/fg_bucketize_model_config.json --label clk @@ -110,7 +110,7 @@ def test_rtp_convert_test_model_config(self): --train_input_path data/test/rtp/taobao_train_feature.txt --eval_input_path data/test/rtp/taobao_test_feature.txt --selected_cols 0,3 --num_steps 400 - """ % pipeline_config_path + """ % pipeline_config_path) proc = test_utils.run_cmd(convert_cmd, '%s/log_%s.txt' % (test_dir, 'convert')) proc.wait() diff --git a/easy_rec/python/test/train_eval_test.py b/easy_rec/python/test/train_eval_test.py index 60745f16f..7e6f5ed50 100644 --- a/easy_rec/python/test/train_eval_test.py +++ b/easy_rec/python/test/train_eval_test.py @@ -15,10 +15,10 @@ from tensorflow.python.platform import gfile from easy_rec.python.main import predict -from easy_rec.python.utils import config_util -from easy_rec.python.utils import constant -from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils import test_utils + +from easy_rec.python.utils import ( # NOQA + config_util, constant, estimator_utils, test_utils, +) try: import graphlearn as gl @@ -95,7 +95,8 @@ def test_deepfm_with_multi_class(self): def test_wide_and_deep_no_final(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/wide_and_deep_no_final_on_avazau_ctr.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_wide_and_deep(self): @@ -107,7 +108,8 @@ def test_wide_and_deep(self): def test_wide_and_deep_backbone(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/wide_and_deep_backbone_on_avazau.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_dlrm(self): @@ -121,13 +123,15 @@ def test_dlrm_backbone(self): def test_adamw_optimizer(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_on_avazu_adamw_ctr.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_momentumw_optimizer(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_on_avazu_momentumw_ctr.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_deepfm_with_param_edit(self): @@ -136,7 +140,8 @@ def test_deepfm_with_param_edit(self): 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', self._test_dir, hyperparam_str='{"model_dir":"%s", ' - '"model_config.deepfm.wide_output_dim": 32}' % model_dir) + '"model_config.deepfm.wide_output_dim": 32}' % model_dir, + ) self.assertTrue(self._success) config_path = os.path.join(model_dir, 'pipeline.config') pipeline_config = config_util.get_configs_from_pipeline_file(config_path) @@ -163,14 +168,16 @@ def test_multi_tower_gauc(self): def test_multi_tower_session_auc(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/multi_tower_on_taobao_session_auc.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_multi_tower_save_checkpoint_secs(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/multi_tower_save_secs_on_taobao.config', self._test_dir, - total_steps=100) + total_steps=100, + ) ckpts_times = [] ckpt_dir = os.path.join(self._test_dir, 'train') for filepath in os.listdir(ckpt_dir): @@ -197,7 +204,8 @@ def _post_check_func(pipeline_config): 'samples/model_config/multi_tower_ckpt_keep_3_on_taobao.config', self._test_dir, total_steps=500, - post_check_func=_post_check_func) + post_check_func=_post_check_func, + ) def test_multi_tower_with_best_exporter(self): @@ -217,7 +225,8 @@ def _post_check_func(pipeline_config): self._test_dir, total_steps=800, post_check_func=_post_check_func, - timeout=3000) + timeout=3000, + ) self.assertTrue(self._success) def test_latest_ckpt(self): @@ -232,13 +241,15 @@ def _post_check_func(pipeline_config): logging.info('model_dir: %s' % pipeline_config.model_dir) logging.info('latest_checkpoint: %s' % estimator_utils.latest_checkpoint(pipeline_config.model_dir)) - return tf.train.latest_checkpoint(pipeline_config.model_dir) == \ - estimator_utils.latest_checkpoint(pipeline_config.model_dir) + return tf.train.latest_checkpoint( + pipeline_config.model_dir) == estimator_utils.latest_checkpoint( + pipeline_config.model_dir) self._success = test_utils.test_single_train_eval( 'samples/model_config/taobao_fg.config', self._test_dir, - post_check_func=_post_check_func) + post_check_func=_post_check_func, + ) self.assertTrue(self._success) def test_oss_stop_signal(self): @@ -262,7 +273,8 @@ def _watch_func(): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/taobao_fg_signal_stop.config', self._test_dir, - total_steps=1000) + total_steps=1000, + ) self.assertTrue(self._success) watch_th.join() final_ckpt = estimator_utils.latest_checkpoint(train_dir) @@ -276,7 +288,8 @@ def test_dead_line_stop_signal(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/dead_line_stop.config', self._test_dir, - total_steps=1000) + total_steps=1000, + ) self.assertTrue(self._success) final_ckpt = estimator_utils.latest_checkpoint(train_dir) ckpt_version = estimator_utils.get_ckpt_version(final_ckpt) @@ -292,21 +305,21 @@ def _post_check_func(pipeline_config): os.path.join(pipeline_config.model_dir, 'pipeline.config'), False) logging.info('fine_tune_checkpoint: %s' % pipeline_config.train_config.fine_tune_checkpoint) - return pipeline_config.train_config.fine_tune_checkpoint == \ - 'data/test/mt_ckpt/model.ckpt-100' + return pipeline_config.train_config.fine_tune_checkpoint == 'data/test/mt_ckpt/model.ckpt-100' self._success = test_utils.test_single_train_eval( 'samples/model_config/multi_tower_on_taobao.config', self._test_dir, fine_tune_checkpoint='data/test/mt_ckpt', - post_check_func=_post_check_func) + post_check_func=_post_check_func, + ) self.assertTrue(self._success) def test_fine_tune_ckpt(self): def _post_check_func(pipeline_config): - pipeline_config.train_config.fine_tune_checkpoint = \ - estimator_utils.latest_checkpoint(pipeline_config.model_dir) + pipeline_config.train_config.fine_tune_checkpoint = estimator_utils.latest_checkpoint( + pipeline_config.model_dir) test_dir = os.path.join(self._test_dir, 'fine_tune') pipeline_config.model_dir = os.path.join(test_dir, 'ckpt') return test_utils.test_single_train_eval(pipeline_config, test_dir) @@ -314,13 +327,15 @@ def _post_check_func(pipeline_config): self._success = test_utils.test_single_train_eval( 'samples/model_config/taobao_fg.config', self._test_dir, - post_check_func=_post_check_func) + post_check_func=_post_check_func, + ) self.assertTrue(self._success) def test_multi_tower_multi_value_export(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/multi_tower_multi_value_export_on_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_multi_tower_fg_input(self): @@ -437,13 +452,15 @@ def test_ppnet(self): def test_uniter_only_text_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/uniter_on_movielens_only_text_feature.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_uniter_only_image_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/uniter_on_movielens_only_image_feature.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_cmbf(self): @@ -459,19 +476,22 @@ def test_cmbf_with_multi_loss(self): def test_cmbf_has_other_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/cmbf_on_movielens_has_other_feature.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_cmbf_only_text_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/cmbf_on_movielens_only_text_feature.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_cmbf_only_image_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/cmbf_on_movielens_only_image_feature.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_dssm(self): @@ -507,21 +527,24 @@ def test_dssm_neg_sampler_v2(self): def test_dssm_hard_neg_sampler(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dssm_hard_neg_sampler_on_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_hard_neg_regular_sampler(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dssm_hard_neg_sampler_regular_on_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_hard_neg_sampler_v2(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dssm_hard_neg_sampler_v2_on_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_dssm_no_norm(self): @@ -572,16 +595,21 @@ def _gen_kd_data(train_path, eval_path): 'kd_pipeline.config') self._success = test_utils.test_single_train_eval( os.path.join(self._test_dir, 'kd_pipeline.config'), - os.path.join(self._test_dir, 'kd')) + os.path.join(self._test_dir, 'kd'), + ) self.assertTrue(self._success) def test_dssm_with_kd(self): - self._test_kd('samples/model_config/multi_tower_on_taobao.config', - 'samples/model_config/dssm_kd_on_taobao.config') + self._test_kd( + 'samples/model_config/multi_tower_on_taobao.config', + 'samples/model_config/dssm_kd_on_taobao.config', + ) def test_deepfm_multi_class_with_kd(self): - self._test_kd('samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', - 'samples/model_config/deepfm_multi_cls_small.config') + self._test_kd( + 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', + 'samples/model_config/deepfm_multi_cls_small.config', + ) def test_mind(self): self._success = test_utils.test_single_train_eval( @@ -601,25 +629,29 @@ def test_deepfm_with_regression(self): def test_deepfm_with_sigmoid_l2_loss(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_on_avazu_sigmoid_l2.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_deepfm_with_embedding_learning_rate(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_on_avazu_emblr_ctr.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_deepfm_with_eval_online(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_on_avazu_eval_online_ctr.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_deepfm_with_eval_online_gauc(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_on_avazu_eval_online_gauc_ctr.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_mmoe(self): @@ -652,7 +684,8 @@ def test_simple_multi_task(self): def test_simple_multi_task_backbone(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/simple_multi_task_backbone_on_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_esmm(self): @@ -693,13 +726,15 @@ def test_dbmtl_uniter(self): def test_dbmtl_with_multi_loss(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_taobao_with_multi_loss.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_early_stop(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/multi_tower_early_stop_on_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_early_stop_custom(self): @@ -711,7 +746,8 @@ def test_early_stop_custom(self): def test_early_stop_dis(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/multi_tower_early_stop_on_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_latest_export_with_asset(self): @@ -734,56 +770,64 @@ def _post_check_func(config): self._success = test_utils.test_single_train_eval( 'samples/model_config/taobao_fg.config', self._test_dir, - post_check_func=_post_check_func) + post_check_func=_post_check_func, + ) self.assertTrue(self._success) def test_dbmtl_variational_dropout(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_variational_dropout.config', self._test_dir, - post_check_func=test_utils.test_feature_selection) + post_check_func=test_utils.test_feature_selection, + ) self.assertTrue(self._success) def test_dbmtl_variational_dropout_feature_num(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_variational_dropout_feature_num.config', self._test_dir, - post_check_func=test_utils.test_feature_selection) + post_check_func=test_utils.test_feature_selection, + ) self.assertTrue(self._success) def test_essm_variational_dropout(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/esmm_variational_dropout_on_taobao.config', self._test_dir, - post_check_func=test_utils.test_feature_selection) + post_check_func=test_utils.test_feature_selection, + ) self.assertTrue(self._success) def test_fm_variational_dropout(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/fm_variational_dropout_on_taobao.config', self._test_dir, - post_check_func=test_utils.test_feature_selection) + post_check_func=test_utils.test_feature_selection, + ) self.assertTrue(self._success) def test_deepfm_with_combo_feature_variational_dropout(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_combo_variational_dropout_on_avazu_ctr.config', self._test_dir, - post_check_func=test_utils.test_feature_selection) + post_check_func=test_utils.test_feature_selection, + ) self.assertTrue(self._success) def test_dbmtl_sequence_variational_dropout(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_variational_dropout_on_sequence_feature_taobao.config', self._test_dir, - post_check_func=test_utils.test_feature_selection) + post_check_func=test_utils.test_feature_selection, + ) self.assertTrue(self._success) def test_din_variational_dropout(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/din_varitional_dropout_on_taobao.config', self._test_dir, - post_check_func=test_utils.test_feature_selection) + post_check_func=test_utils.test_feature_selection, + ) self.assertTrue(self._success) def test_rocket_launching(self): @@ -800,7 +844,8 @@ def test_rocket_launching_feature_based(self): def test_rocket_launching_with_rtp_input(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/rocket_launching_with_rtp_input.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_dbmtl_mmoe(self): @@ -813,14 +858,15 @@ def test_train_with_ps_worker(self): 'samples/model_config/multi_tower_on_taobao.config', self._test_dir) self.assertTrue(self._success) - @unittest.skip("Timeout on CI machine") + @unittest.skip('Timeout on CI machine') def test_fit_on_eval(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/multi_tower_on_taobao.config', self._test_dir, total_steps=10, num_evaluator=1, - fit_on_eval=True) + fit_on_eval=True, + ) self.assertTrue(self._success) def test_unbalance_data(self): @@ -829,14 +875,16 @@ def test_unbalance_data(self): self._test_dir, total_steps=0, num_epoch=1, - num_evaluator=1) + num_evaluator=1, + ) self.assertTrue(self._success) def test_train_with_ps_worker_with_evaluator(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/multi_tower_on_taobao.config', self._test_dir, - num_evaluator=1) + num_evaluator=1, + ) self.assertTrue(self._success) final_export_dir = os.path.join(self._test_dir, 'train/export/final') all_saved_files = glob.glob(final_export_dir + '/*/saved_model.pb') @@ -847,7 +895,8 @@ def test_train_with_ps_worker_with_evaluator(self): def test_train_with_ps_worker_chief_redundant(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/multi_tower_on_taobao_chief_redundant.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_deepfm_embed_input(self): @@ -868,7 +917,8 @@ def test_tfrecord_input(self): def test_batch_tfrecord_input(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/deepfm_on_criteo_batch_tfrecord.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_autodis_embedding(self): @@ -897,25 +947,30 @@ def test_dssm_sample_weight(self): def test_dssm_neg_sampler_with_sample_weight(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dssm_neg_sampler_with_sample_weight.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) @unittest.skipIf( LooseVersion(tf.__version__) != LooseVersion('2.3.0'), - 'MultiWorkerMirroredStrategy need tf version == 2.3') + 'MultiWorkerMirroredStrategy need tf version == 2.3', + ) def test_train_with_multi_worker_mirror(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/multi_tower_multi_worker_mirrored_strategy_on_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) @unittest.skipIf( LooseVersion(tf.__version__) != LooseVersion('2.3.0'), - 'MultiWorkerMirroredStrategy need tf version == 2.3') + 'MultiWorkerMirroredStrategy need tf version == 2.3', + ) def test_train_mmoe_with_multi_worker_mirror(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/mmoe_mirrored_strategy_on_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_fg_dtype(self): @@ -932,13 +987,15 @@ def test_share_not_used(self): def test_sequence_autoint(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/autoint_on_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_sequence_dcn(self): @@ -950,19 +1007,22 @@ def test_sequence_dcn(self): def test_sequence_dssm(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dssm_on_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_sequence_esmm(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/esmm_on_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_sequence_mmoe(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/mmoe_on_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_sequence_ple(self): @@ -974,73 +1034,85 @@ def test_sequence_ple(self): def test_sequence_rocket_launching(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/rocket_launching_on_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_sequence_simple_multi_task(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/simple_multi_task_on_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_sequence_wide_and_deep(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/wide_and_deep_on_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_numeric_boundary_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_numeric_boundary_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_numeric_hash_bucket_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_numeric_hash_bucket_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_numeric_raw_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_numeric_raw_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_numeric_num_buckets_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_numeric_num_buckets_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_multi_numeric_boundary_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_multi_numeric_boundary_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_multi_numeric_hash_bucket_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_multi_numeric_hash_bucket_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_multi_numeric_raw_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_multi_numeric_raw_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_multi_numeric_num_buckets_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_multi_numeric_num_buckets_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_multi_sequence_dbmtl(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_multi_sequence_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_multi_optimizer(self): @@ -1051,7 +1123,8 @@ def test_multi_optimizer(self): def test_embedding_separate_optimizer(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/deepfm_combo_on_avazu_embed_adagrad.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_expr_feature(self): @@ -1072,15 +1145,16 @@ def _post_check_config(pipeline_config): 'train/pipeline.config') pipeline_config = config_util.get_configs_from_pipeline_file( train_saved_config_path) - assert pipeline_config.model_config.deepfm.wide_output_dim == 8,\ - 'invalid model_config.deepfm.wide_output_dim=%d' % \ - pipeline_config.model_config.deepfm.wide_output_dim + assert pipeline_config.model_config.deepfm.wide_output_dim == 8, ( + 'invalid model_config.deepfm.wide_output_dim=%d' % + pipeline_config.model_config.deepfm.wide_output_dim) self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', self._test_dir, post_check_func=_post_check_config, - extra_cmd_args='--model_config.deepfm.wide_output_dim 8') + extra_cmd_args='--model_config.deepfm.wide_output_dim 8', + ) def test_cmd_config_param_v2(self): @@ -1089,15 +1163,16 @@ def _post_check_config(pipeline_config): 'train/pipeline.config') pipeline_config = config_util.get_configs_from_pipeline_file( train_saved_config_path) - assert pipeline_config.model_config.deepfm.wide_output_dim == 1,\ - 'invalid model_config.deepfm.wide_output_dim=%d' % \ - pipeline_config.model_config.deepfm.wide_output_dim + assert pipeline_config.model_config.deepfm.wide_output_dim == 1, ( + 'invalid model_config.deepfm.wide_output_dim=%d' % + pipeline_config.model_config.deepfm.wide_output_dim) self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', self._test_dir, post_check_func=_post_check_config, - extra_cmd_args='--model_config.deepfm.wide_output_dim=1') + extra_cmd_args='--model_config.deepfm.wide_output_dim=1', + ) def test_cmd_config_param_v3(self): @@ -1106,56 +1181,69 @@ def _post_check_config(pipeline_config): 'train/pipeline.config') pipeline_config = config_util.get_configs_from_pipeline_file( train_saved_config_path) - assert pipeline_config.model_config.deepfm.wide_output_dim == 3,\ - 'invalid model_config.deepfm.wide_output_dim=%d' % \ - pipeline_config.model_config.deepfm.wide_output_dim + assert pipeline_config.model_config.deepfm.wide_output_dim == 3, ( + 'invalid model_config.deepfm.wide_output_dim=%d' % + pipeline_config.model_config.deepfm.wide_output_dim) self._success = test_utils.test_single_train_eval( 'samples/model_config/deepfm_multi_cls_on_avazu_ctr.config', self._test_dir, post_check_func=_post_check_config, - extra_cmd_args='--model_config.deepfm.wide_output_dim="3"') + extra_cmd_args='--model_config.deepfm.wide_output_dim="3"', + ) def test_distribute_eval_deepfm_multi_cls(self): cur_eval_path = 'data/test/distribute_eval_test/deepfm_distribute_eval_dwd_avazu_out_multi_cls' self._success = test_utils.test_distributed_eval( 'samples/model_config/deepfm_distribute_eval_multi_cls_on_avazu_ctr.config', - cur_eval_path, self._test_dir) + cur_eval_path, + self._test_dir, + ) self.assertTrue(self._success) def test_distribute_eval_deepfm_single_cls(self): cur_eval_path = 'data/test/distribute_eval_test/dwd_distribute_eval_avazu_out_test_combo' self._success = test_utils.test_distributed_eval( 'samples/model_config/deepfm_distribute_eval_combo_on_avazu_ctr.config', - cur_eval_path, self._test_dir) + cur_eval_path, + self._test_dir, + ) self.assertTrue(self._success) def test_distribute_eval_dssm_pointwise_classification(self): cur_eval_path = 'data/test/distribute_eval_test/dssm_distribute_eval_pointwise_classification_taobao_ckpt' self._success = test_utils.test_distributed_eval( 'samples/model_config/dssm_distribute_eval_pointwise_classification_on_taobao.config', - cur_eval_path, self._test_dir) + cur_eval_path, + self._test_dir, + ) self.assertTrue(self._success) def test_distribute_eval_dssm_reg(self): cur_eval_path = 'data/test/distribute_eval_test/dssm_distribute_eval_reg_taobao_ckpt' self._success = test_utils.test_distributed_eval( 'samples/model_config/dssm_distribute_eval_reg_on_taobao.config', - cur_eval_path, self._test_dir) + cur_eval_path, + self._test_dir, + ) self.assertTrue(self._success) def test_distribute_eval_dropout(self): cur_eval_path = 'data/test/distribute_eval_test/dropoutnet_distribute_eval_taobao_ckpt' self._success = test_utils.test_distributed_eval( 'samples/model_config/dropoutnet_distribute_eval_on_taobao.config', - cur_eval_path, self._test_dir) + cur_eval_path, + self._test_dir, + ) self.assertTrue(self._success) def test_distribute_eval_esmm(self): cur_eval_path = 'data/test/distribute_eval_test/esmm_distribute_eval_taobao_ckpt' self._success = test_utils.test_distributed_eval( 'samples/model_config/esmm_distribute_eval_on_taobao.config', - cur_eval_path, self._test_dir) + cur_eval_path, + self._test_dir, + ) self.assertTrue(self._success) def test_share_no_used(self): @@ -1167,46 +1255,53 @@ def test_share_no_used(self): def test_dssm_neg_sampler_sequence_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dssm_neg_sampler_sequence_feature.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_dssm_neg_sampler_need_key_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dssm_neg_sampler_need_key_feature.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_dbmtl_on_multi_numeric_boundary_need_key_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_multi_numeric_boundary_need_key_feature_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_dbmtl_on_multi_numeric_boundary_allow_key_transform(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_multi_numeric_boundary_allow_key_transform.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_dbmtl_on_multi_numeric_boundary_aux_hist_seq(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/dbmtl_on_numeric_boundary_sequence_feature_aux_hist_seq_taobao.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_multi_tower_recall_neg_sampler_sequence_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/multi_tower_recall_neg_sampler_sequence_feature.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) @unittest.skipIf(gl is None, 'graphlearn is not installed') def test_multi_tower_recall_neg_sampler_only_sequence_feature(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/multi_tower_recall_neg_sampler_only_sequence_feature.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) @unittest.skipIf(hvd is None, 'horovod is not installed') @@ -1214,7 +1309,8 @@ def test_horovod(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/deepfm_combo_on_avazu_ctr.config', self._test_dir, - use_hvd=True) + use_hvd=True, + ) self.assertTrue(self._success) @unittest.skipIf(hvd is None or sok is None, @@ -1223,7 +1319,8 @@ def test_sok(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/multi_tower_on_taobao_sok.config', self._test_dir, - use_hvd=True) + use_hvd=True, + ) self.assertTrue(self._success) @unittest.skipIf( @@ -1240,7 +1337,8 @@ def test_train_parquet_embedding_parallel(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/dlrm_on_criteo_parquet_ep.config', self._test_dir, - use_hvd=True) + use_hvd=True, + ) self.assertTrue(self._success) @unittest.skipIf(hvd is None, 'horovod is not installed') @@ -1248,7 +1346,8 @@ def test_train_parquet_embedding_parallel_v2(self): self._success = test_utils.test_distributed_train_eval( 'samples/model_config/dlrm_on_criteo_parquet_ep_v2.config', self._test_dir, - use_hvd=True) + use_hvd=True, + ) self.assertTrue(self._success) def test_pdn(self): @@ -1279,7 +1378,8 @@ def test_dssm_senet_backbone_on_taobao(self): def test_parallel_dssm_backbone_on_taobao(self): self._success = test_utils.test_single_train_eval( 'samples/model_config/parallel_dssm_on_taobao_backbone.config', - self._test_dir) + self._test_dir, + ) self.assertTrue(self._success) def test_xdeefm_backbone_on_taobao(self): diff --git a/easy_rec/python/test/util_test.py b/easy_rec/python/test/util_test.py index c14524488..d3a0dbb70 100644 --- a/easy_rec/python/test/util_test.py +++ b/easy_rec/python/test/util_test.py @@ -49,9 +49,12 @@ def test_get_expression_and(self): result = get_expression( '(age_level>item_age_level) & (age_level3)|(item_age_level<1)', diff --git a/easy_rec/python/test/zero_inflated_lognormal_test.py b/easy_rec/python/test/zero_inflated_lognormal_test.py index f512e48e8..f88f481fe 100644 --- a/easy_rec/python/test/zero_inflated_lognormal_test.py +++ b/easy_rec/python/test/zero_inflated_lognormal_test.py @@ -4,47 +4,62 @@ import tensorflow as tf from scipy import stats -from easy_rec.python.loss.zero_inflated_lognormal import zero_inflated_lognormal_loss # NOQA +from easy_rec.python.loss.zero_inflated_lognormal import ( # NOQA + zero_inflated_lognormal_loss,) if tf.__version__ >= '2.0': tf = tf.compat.v1 -# Absolute error tolerance in asserting array near. -_ERR_TOL = 1e-6 +class ZeroInflatedLognormalLossTest(tf.test.TestCase): -# softplus function that calculates log(1+exp(x)) -def _softplus(x): - return np.log(1.0 + np.exp(x)) + def setUp(self): + super(ZeroInflatedLognormalLossTest, self).setUp() + self.logits = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]) + self.labels = np.array([[0.0], [1.5]]) + def zero_inflated_lognormal(self, labels, logits, max_sigma=5.0): + labels = labels.astype(np.float32) + if labels.ndim == 1: + labels = labels[:, None] + logits = logits.astype(np.float32) + positive = (labels > 0).astype(np.float32) # [B,1] -# sigmoid function that calculates 1/(1+exp(-x)) -def _sigmoid(x): - return 1 / (1 + np.exp(-x)) + z = logits[..., :1] + mu = logits[..., 1:2] + sigma = np.log1p(np.exp(logits[..., 2:])) # softplus + sigma = np.clip(sigma, np.sqrt(tf.keras.backend.epsilon()), max_sigma) + # 分类损失：BCE with logits，按 batch 平均 + # bce(z, y) = max(z,0) - z*y + log(1 + exp(-|z|)) + bce = np.maximum(z, 0.0) - z * positive + np.log1p(np.exp(-np.abs(z))) + classification_loss = np.mean(bce) -class ZeroInflatedLognormalLossTest(tf.test.TestCase): - - def setUp(self): - super(ZeroInflatedLognormalLossTest, self).setUp() - self.logits = np.array([[.1, .2, .3], [.4, .5, .6]]) - self.labels = np.array([[0.], [1.5]]) - - def zero_inflated_lognormal(self, labels, logits): - positive_logits = logits[..., :1] - loss_zero = _softplus(positive_logits) - loc = logits[..., 1:2] - scale = np.maximum( - _softplus(logits[..., 2:]), np.sqrt(tf.keras.backend.epsilon())) - log_prob_non_zero = stats.lognorm.logpdf( - x=labels, s=scale, loc=0, scale=np.exp(loc)) - loss_non_zero = _softplus(-positive_logits) - log_prob_non_zero - return np.mean(np.where(labels == 0., loss_zero, loss_non_zero), axis=-1) + # 回归损失：仅对正样本的 log_prob 求平均（按正样本数） + # 避免对 y=0 求 logpdf + mask = (labels > 0).squeeze(-1) + logprob = np.zeros_like(labels, dtype=np.float64) + if np.any(mask): + # scipy 参数化：s=shape= sigma, scale=exp(mu), loc=0 + logprob[mask, 0] = stats.lognorm.logpdf( + x=labels[mask, 0].astype(np.float64), + s=sigma[mask, 0].astype(np.float64), + loc=0.0, + scale=np.exp(mu[mask, 0].astype(np.float64)), + ) + num_pos = np.sum(positive) + 1e-8 + regression_loss = -(np.sum(positive * logprob) / num_pos) + # 与实现一致的组合（此处测试会把正则项设为0） + total = classification_loss + regression_loss + return float(total) def test_loss_value(self): expected_loss = self.zero_inflated_lognormal(self.labels, self.logits) expected_loss = np.average(expected_loss) - loss = zero_inflated_lognormal_loss(self.labels, self.logits) + loss = zero_inflated_lognormal_loss( + self.labels, self.logits, mu_reg=0, sigma_reg=0) + # Absolute error tolerance in asserting array near. + _ERR_TOL = 1e-6 self.assertNear(self.evaluate(loss), expected_loss, _ERR_TOL) diff --git a/easy_rec/python/tools/add_boundaries_to_config.py b/easy_rec/python/tools/add_boundaries_to_config.py index 18d5f6037..f78b94e46 100644 --- a/easy_rec/python/tools/add_boundaries_to_config.py +++ b/easy_rec/python/tools/add_boundaries_to_config.py @@ -16,7 +16,8 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + level=logging.INFO, +) tf.app.flags.DEFINE_string('template_config_path', None, 'Path to template pipeline config ' 'file.') diff --git a/easy_rec/python/tools/add_feature_info_to_config.py b/easy_rec/python/tools/add_feature_info_to_config.py index 7594d038b..d525bf893 100644 --- a/easy_rec/python/tools/add_feature_info_to_config.py +++ b/easy_rec/python/tools/add_feature_info_to_config.py @@ -16,7 +16,8 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + level=logging.INFO, +) tf.app.flags.DEFINE_string('template_config_path', None, 'Path to template pipeline config ' 'file.') @@ -40,7 +41,8 @@ def main(argv): data_config=pipeline_config.data_config, hive_config=pipeline_config.hive_train_input, selected_cols=sels, - record_defaults=['', '', '']) + record_defaults=['', '', ''], + ) reader = hive_util.hive_read_line(FLAGS.config_table) for record in reader: feature_name = record[0][0] @@ -50,6 +52,7 @@ def main(argv): else: import common_io + reader = common_io.table.TableReader(FLAGS.config_table, selected_cols=sels) while True: try: diff --git a/easy_rec/python/tools/convert_rtp_data.py b/easy_rec/python/tools/convert_rtp_data.py index 18455bd9c..97513231d 100644 --- a/easy_rec/python/tools/convert_rtp_data.py +++ b/easy_rec/python/tools/convert_rtp_data.py @@ -9,6 +9,7 @@ In our new format: ...beautysmart... """ + import argparse import csv import json @@ -19,7 +20,8 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + level=logging.INFO, +) if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/tools/convert_rtp_fg.py b/easy_rec/python/tools/convert_rtp_fg.py index bd090e0a3..cfe7434bd 100644 --- a/easy_rec/python/tools/convert_rtp_fg.py +++ b/easy_rec/python/tools/convert_rtp_fg.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Convert rtp fg feature config to easy_rec data_config and feature_config.""" + import argparse import logging import sys @@ -12,7 +13,8 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + level=logging.INFO, +) if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -25,7 +27,8 @@ type=str, choices=model_types, default='', - help='model type, currently support: %s' % ','.join(model_types)) + help='model type, currently support: %s' % ','.join(model_types), + ) parser.add_argument('--rtp_fg', type=str, help='rtp fg path') parser.add_argument( '--embedding_dim', type=int, default=16, help='embedding_dimension') @@ -42,19 +45,21 @@ '--num_steps', type=int, default=1000, - help='number of train steps = num_samples * num_epochs / batch_size / num_workers' + help='number of train steps = num_samples * num_epochs / batch_size / num_workers', ) parser.add_argument('--output_path', type=str, help='generated config path') parser.add_argument( '--incol_separator', type=str, default='\003', - help='separator for multi_value features') + help='separator for multi_value features', + ) parser.add_argument( '--separator', type=str, default='\002', - help='separator between different features') + help='separator between different features', + ) parser.add_argument( '--train_input_path', type=str, default=None, help='train data path') parser.add_argument( @@ -64,7 +69,7 @@ type=str, default=None, help='selected cols, for csv input, it is in the format of: label_col_id0,...,lable_cold_idn,feature_col_id ' - 'for odps table input, it is in the format of: label_col_name0,...,label_col_namen,feature_col_name ' + 'for odps table input, it is in the format of: label_col_name0,...,label_col_namen,feature_col_name ', ) parser.add_argument( '--rtp_separator', type=str, default=';', help='separator') @@ -72,7 +77,8 @@ '--input_type', type=str, default='OdpsRTPInput', - help='default to OdpsRTPInput, if test local, change it to RTPInput') + help='default to OdpsRTPInput, if test local, change it to RTPInput', + ) parser.add_argument( '--is_async', action='store_true', help='async mode, debug to false') @@ -86,12 +92,21 @@ logging.error('output_path is not set') sys.exit(1) - pipeline_config = convert_rtp_fg(args.rtp_fg, args.embedding_dim, - args.batch_size, args.label, args.num_steps, - args.model_type, args.separator, - args.incol_separator, args.train_input_path, - args.eval_input_path, args.selected_cols, - args.input_type, args.is_async) + pipeline_config = convert_rtp_fg( + args.rtp_fg, + args.embedding_dim, + args.batch_size, + args.label, + args.num_steps, + args.model_type, + args.separator, + args.incol_separator, + args.train_input_path, + args.eval_input_path, + args.selected_cols, + args.input_type, + args.is_async, + ) save_message(pipeline_config, args.output_path) logging.info('Conversion done.') logging.info('Tips:') diff --git a/easy_rec/python/tools/create_config_from_excel.py b/easy_rec/python/tools/create_config_from_excel.py index c0f3e0a2b..cf31bdb1b 100644 --- a/easy_rec/python/tools/create_config_from_excel.py +++ b/easy_rec/python/tools/create_config_from_excel.py @@ -15,8 +15,17 @@ class ModelConfigConverter: - def __init__(self, excel_path, output_path, model_type, column_separator, - incol_separator, train_input_path, eval_input_path, model_dir): + def __init__( + self, + excel_path, + output_path, + model_type, + column_separator, + incol_separator, + train_input_path, + eval_input_path, + model_dir, + ): self._excel_path = excel_path self._output_path = output_path self._model_type = model_type @@ -40,7 +49,7 @@ def _get_type_name(self, input_name): 'double': 'DOUBLE', 'float': 'FLOAT', 'string': 'STRING', - 'bool': 'BOOL' + 'bool': 'BOOL', } return type_dict[input_name] @@ -50,7 +59,7 @@ def _get_type_default(self, input_name): 'double': '0.0', 'float': '0.0', 'string': '', - 'bool': 'false' + 'bool': 'false', } return type_dict[input_name] @@ -139,8 +148,13 @@ def _is_good(v): field['embedding_name'] = field['global'] for t in [ - 'type', 'global', 'hash_bucket_size', 'embedding_dim', - 'default_value', 'weights', 'boundaries' + 'type', + 'global', + 'hash_bucket_size', + 'embedding_dim', + 'default_value', + 'weights', + 'boundaries', ]: if t not in row: continue @@ -370,7 +384,7 @@ def convert(self): self._write_multi_tower_config(fout) else: logging.warning( - 'the model_config could not be generated automatically, you have to write the model_config manually.' + 'the model_config could not be generated automatically, you have to write the model_config manually' ) # reformat the config pipeline_config = config_util.get_configs_from_pipeline_file( @@ -388,19 +402,22 @@ def convert(self): '--model_type', type=str, choices=model_types, - help='model type, currently support: %s' % ','.join(model_types)) + help='model type, currently support: %s' % ','.join(model_types), + ) parser.add_argument('--excel_path', type=str, help='excel config path') parser.add_argument('--output_path', type=str, help='generated config path') parser.add_argument( '--column_separator', type=str, default=',', - help='column separator, separator betwen features') + help='column separator, separator between features', + ) parser.add_argument( '--incol_separator', type=str, default='|', - help='separator within features, such as tag features') + help='separator within features, such as tag features', + ) parser.add_argument( '--train_input_path', type=str, default='', help='train input path') parser.add_argument( @@ -415,10 +432,16 @@ def convert(self): logging.info('column_separator = %s in_column_separator = %s' % (args.column_separator, args.incol_separator)) - converter = ModelConfigConverter(args.excel_path, args.output_path, - args.model_type, args.column_separator, - args.incol_separator, args.train_input_path, - args.eval_input_path, args.model_dir) + converter = ModelConfigConverter( + args.excel_path, + args.output_path, + args.model_type, + args.column_separator, + args.incol_separator, + args.train_input_path, + args.eval_input_path, + args.model_dir, + ) converter.convert() logging.info('Conversion done') logging.info('Tips:') diff --git a/easy_rec/python/tools/criteo/convert_data.py b/easy_rec/python/tools/criteo/convert_data.py index 6382865e7..222a25a75 100644 --- a/easy_rec/python/tools/criteo/convert_data.py +++ b/easy_rec/python/tools/criteo/convert_data.py @@ -75,8 +75,12 @@ def convert(input_path, prefix, part_record_num, save_format): total_line += sid sid = 0 if sid > 0: - save_func(labels[:sid], dense_arr[:sid], cate_arr[:sid], - prefix + '_' + str(part_id)) + save_func( + labels[:sid], + dense_arr[:sid], + cate_arr[:sid], + prefix + '_' + str(part_id), + ) logging.info('\t%s write final part: %d' % (input_path, part_id)) part_id += 1 total_line += sid @@ -115,17 +119,20 @@ def convert(input_path, prefix, part_record_num, save_format): '--save_format', type=str, default='npy', - help='save format, choices: npy|parquet') + help='save format, choices: npy|parquet', + ) parser.add_argument( '--part_record_num', type=int, default=1024 * 1024 * 8, - help='the maximal number of samples in each binary file') + help='the maximal number of samples in each binary file', + ) parser.add_argument( '--dt', nargs='*', type=int, - help='select days to convert, default to select all: 0-23') + help='select days to convert, default to select all: 0-23', + ) args = parser.parse_args() @@ -149,7 +156,8 @@ def convert(input_path, prefix, part_record_num, save_format): prefix = os.path.join(args.save_dir, str(d)) proc = multiprocessing.Process( target=convert, - args=(input_path, prefix, args.part_record_num, args.save_format)) + args=(input_path, prefix, args.part_record_num, args.save_format), + ) convert(input_path, prefix, args.part_record_num, args.save_format) proc.start() proc_arr.append(proc) diff --git a/easy_rec/python/tools/edit_lookup_graph.py b/easy_rec/python/tools/edit_lookup_graph.py index 7f1fc1fbf..284087772 100644 --- a/easy_rec/python/tools/edit_lookup_graph.py +++ b/easy_rec/python/tools/edit_lookup_graph.py @@ -16,7 +16,8 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + level=logging.INFO, +) if __name__ == '__main__': """Replace the default embedding_lookup ops with self defined embedding lookup ops. @@ -65,7 +66,8 @@ args.redis_passwd, args.time_out, meta_graph_def=None, - debug_dir=args.output_dir if args.verbose else '') + debug_dir=args.output_dir if args.verbose else '', + ) meta_graph_editor.edit_graph() meta_graph_version = meta_graph_editor.meta_graph_version @@ -88,7 +90,8 @@ tf.add_to_collection( tf.GraphKeys.ASSET_FILEPATHS, tf.constant( - embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt')) + embed_name_to_id_file, dtype=tf.string, name='embed_name_to_ids.txt'), + ) graph = tf.get_default_graph() inputs = meta_graph_editor.signature_def.inputs diff --git a/easy_rec/python/tools/feature_selection.py b/easy_rec/python/tools/feature_selection.py index f50a00fac..491d0f6fb 100644 --- a/easy_rec/python/tools/feature_selection.py +++ b/easy_rec/python/tools/feature_selection.py @@ -18,6 +18,7 @@ tf = tf.compat.v1 import matplotlib # NOQA + matplotlib.use('Agg') # NOQA import matplotlib.pyplot as plt # NOQA @@ -39,13 +40,15 @@ class VariationalDropoutFS: - def __init__(self, - config_path, - output_dir, - topk, - checkpoint_path=None, - fg_path=None, - visualize=False): + def __init__( + self, + config_path, + output_dir, + topk, + checkpoint_path=None, + fg_path=None, + visualize=False, + ): self._config_path = config_path self._output_dir = output_dir self._topk = topk @@ -57,8 +60,10 @@ def __init__(self, def process(self): tf.logging.info('Loading logit_p of VariationalDropout layer ...') - feature_dim_dropout_p_map, embedding_wise_variational_dropout = self._feature_dim_dropout_ratio( - ) + ( + feature_dim_dropout_p_map, + embedding_wise_variational_dropout, + ) = self._feature_dim_dropout_ratio() feature_importance_map = {} for group_name, feature_dim_dropout_p in feature_dim_dropout_p_map.items(): @@ -179,8 +184,9 @@ def _process_config(self, feature_importance_map): feature_configs = [] for feature_config in config_util.get_compatible_feature_configs(config): - feature_name = feature_config.feature_name if feature_config.HasField('feature_name') \ - else feature_config.input_names[0] + feature_name = ( + feature_config.feature_name if feature_config.HasField('feature_name') + else feature_config.input_names[0]) if feature_name not in excluded_features: feature_configs.append(feature_config) @@ -222,10 +228,13 @@ def _dump_to_csv(self, feature_importance, group_name): """Dump feature importance data to a csv file.""" with tf.gfile.Open( os.path.join(self._output_dir, - 'feature_dropout_ratio_%s.csv' % group_name), 'w') as f: + 'feature_dropout_ratio_%s.csv' % group_name), + 'w', + ) as f: df = pd.DataFrame( columns=['feature_name', 'mean_drop_p'], - data=[list(kv) for kv in feature_importance.items()]) + data=[list(kv) for kv in feature_importance.items()], + ) df.to_csv(f, encoding='gbk') def _visualize_embedding_dim_importance(self, feature_dim_dropout_p): @@ -251,7 +260,8 @@ def _visualize_embedding_dim_importance(self, feature_dim_dropout_p): align='center', alpha=0.4, label='dropout_rate', - lw=1) + lw=1, + ) for rect in b: w = rect.get_width() ax.text( @@ -259,7 +269,8 @@ def _visualize_embedding_dim_importance(self, feature_dim_dropout_p): rect.get_y() + rect.get_height() / 2, '%.4f' % w, ha='left', - va='center') + va='center', + ) plt.yticks(y_pos, embedding_dims) plt.xlabel(feature_name) plt.title('Dropout ratio') @@ -271,7 +282,8 @@ def _visualize_feature_importance(self, feature_importance, group_name): """Draw feature importance histogram.""" df = pd.DataFrame( columns=['feature_name', 'mean_drop_p'], - data=[list(kv) for kv in feature_importance.items()]) + data=[list(kv) for kv in feature_importance.items()], + ) df['color'] = ['red' if x < 0.5 else 'green' for x in df['mean_drop_p']] df.sort_values('mean_drop_p', inplace=True, ascending=False) df.reset_index(inplace=True) @@ -288,7 +300,8 @@ def _visualize_feature_importance(self, feature_importance, group_name): fontdict={ 'color': 'red' if x < 0 else 'green', 'size': 14 - }) + }, + ) # Decorations plt.yticks(df.index, df.feature_name, fontsize=20) plt.title('Dropout Ratio', fontdict={'size': 30}) @@ -296,7 +309,9 @@ def _visualize_feature_importance(self, feature_importance, group_name): plt.xlim(0, 1) with tf.gfile.GFile( os.path.join(self._output_dir, - 'feature_dropout_pic_%s.png' % group_name), 'wb') as f: + 'feature_dropout_pic_%s.png' % group_name), + 'wb', + ) as f: plt.savefig(f, format='png') @@ -309,7 +324,8 @@ def _visualize_feature_importance(self, feature_importance, group_name): FLAGS.topk, checkpoint_path=FLAGS.checkpoint_path, fg_path=FLAGS.fg_path, - visualize=FLAGS.visualize) + visualize=FLAGS.visualize, + ) fs.process() else: raise ValueError('Unknown feature selection model type %s' % diff --git a/easy_rec/python/tools/hit_rate_ds.py b/easy_rec/python/tools/hit_rate_ds.py index 5528e0aa2..24c0b6a73 100644 --- a/easy_rec/python/tools/hit_rate_ds.py +++ b/easy_rec/python/tools/hit_rate_ds.py @@ -29,19 +29,22 @@ from easy_rec.python.utils import config_util from easy_rec.python.utils import io_util from easy_rec.python.utils.config_util import process_multi_file_input_path -from easy_rec.python.utils.hit_rate_utils import compute_hitrate_batch -from easy_rec.python.utils.hit_rate_utils import load_graph -from easy_rec.python.utils.hit_rate_utils import reduce_hitrate from easy_rec.python.utils.hive_utils import HiveUtils +from easy_rec.python.utils.hit_rate_utils import ( # NOQA + compute_hitrate_batch, load_graph, reduce_hitrate, +) + if tf.__version__ >= '2.0': tf = tf.compat.v1 -from easy_rec.python.utils.distribution_utils import set_tf_config_and_get_train_worker_num_on_ds # NOQA +from easy_rec.python.utils.distribution_utils import ( # NOQA + set_tf_config_and_get_train_worker_num_on_ds,) logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + level=logging.INFO, +) tf.app.flags.DEFINE_string('item_emb_table', '', 'item embedding table name') tf.app.flags.DEFINE_string('gt_table', '', 'ground truth table name') @@ -85,8 +88,17 @@ def compute_hitrate(g, gt_all, hitrate_writer, gt_table=None): for gt_record in gt_all: gt_record = list(gt_record) - hits, gt_count, src_ids, recall_ids, recall_distances, hitrates, bad_cases, bad_dists = \ - compute_hitrate_batch(g, gt_record, FLAGS.emb_dim, FLAGS.num_interests, FLAGS.top_k) + ( + hits, + gt_count, + src_ids, + recall_ids, + recall_distances, + hitrates, + bad_cases, + bad_dists, + ) = compute_hitrate_batch(g, gt_record, FLAGS.emb_dim, FLAGS.num_interests, + FLAGS.top_k) total_hits += hits total_gt_count += gt_count @@ -103,8 +115,14 @@ def compute_hitrate(g, gt_all, hitrate_writer, gt_table=None): bad_dists = [','.join(str(x) for x in dist) for dist in bad_dists] hitrate_writer.write('\n'.join([ - '\t'.join(line) for line in zip(src_ids, topk_recalls, topk_dists, - hitrates, bad_cases, bad_dists) + '\t'.join(line) for line in zip( + src_ids, + topk_recalls, + topk_dists, + hitrates, + bad_cases, + bad_dists, + ) ])) print('total_hits: ', total_hits) print('total_gt_count: ', total_gt_count) @@ -112,7 +130,6 @@ def compute_hitrate(g, gt_all, hitrate_writer, gt_table=None): def gt_hdfs(gt_table, batch_size, gt_file_sep): - if '*' in gt_table or ',' in gt_table: file_paths = tf.gfile.Glob(gt_table.split(',')) elif tf.gfile.IsDirectory(gt_table): @@ -158,7 +175,8 @@ def main(): else: hive_utils = HiveUtils( data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input) + hive_config=pipeline_config.hive_train_input, + ) i_emb_table = hive_utils.get_table_location(i_emb_table) g = load_graph(i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, @@ -189,7 +207,8 @@ def main(): gt_reader = HiveUtils( data_config=pipeline_config.data_config, hive_config=pipeline_config.hive_train_input, - selected_cols='*') + selected_cols='*', + ) gt_all = gt_reader.hive_read_lines(gt_table, FLAGS.batch_size) if not tf.gfile.IsDirectory(hitrate_details_result): tf.gfile.MakeDirs(hitrate_details_result) diff --git a/easy_rec/python/tools/hit_rate_pai.py b/easy_rec/python/tools/hit_rate_pai.py index 977df20be..4db7795f1 100644 --- a/easy_rec/python/tools/hit_rate_pai.py +++ b/easy_rec/python/tools/hit_rate_pai.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================= """Evaluation of Top k hitrate.""" + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -22,9 +23,10 @@ import tensorflow as tf from easy_rec.python.utils import io_util -from easy_rec.python.utils.hit_rate_utils import compute_hitrate_batch -from easy_rec.python.utils.hit_rate_utils import load_graph -from easy_rec.python.utils.hit_rate_utils import reduce_hitrate + +from easy_rec.python.utils.hit_rate_utils import ( # NOQA + compute_hitrate_batch, load_graph, reduce_hitrate, +) flags = tf.app.flags FLAGS = flags.FLAGS @@ -62,8 +64,17 @@ def compute_hitrate(g, gt_reader, hitrate_writer): while True: try: gt_record = gt_reader.read(FLAGS.batch_size) - hits, gt_count, src_ids, recall_ids, recall_distances, hitrates, bad_cases, bad_dists = \ - compute_hitrate_batch(g, gt_record, FLAGS.emb_dim, FLAGS.num_interests, FLAGS.top_k) + ( + hits, + gt_count, + src_ids, + recall_ids, + recall_distances, + hitrates, + bad_cases, + bad_dists, + ) = compute_hitrate_batch(g, gt_record, FLAGS.emb_dim, + FLAGS.num_interests, FLAGS.top_k) total_hits += hits total_gt_count += gt_count topk_recalls = [','.join(str(x) for x in ids) for ids in recall_ids] @@ -75,9 +86,16 @@ def compute_hitrate(g, gt_reader, hitrate_writer): hitrate_writer.write( list( - zip(src_ids, topk_recalls, topk_dists, hitrates, bad_cases, - bad_dists)), - indices=[0, 1, 2, 3, 4, 5]) + zip( + src_ids, + topk_recalls, + topk_dists, + hitrates, + bad_cases, + bad_dists, + )), + indices=[0, 1, 2, 3, 4, 5], + ) except tf.python_io.OutOfRangeException: break return total_hits, total_gt_count @@ -88,12 +106,22 @@ def main(): input_tables = FLAGS.tables.split(',') if FLAGS.recall_type == 'u2i': i_emb_table, gt_table = input_tables - g = load_graph(i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, - FLAGS.knn_strict) + g = load_graph( + i_emb_table, + FLAGS.emb_dim, + FLAGS.knn_metric, + FLAGS.timeout, + FLAGS.knn_strict, + ) else: i_emb_table, gt_table = input_tables[-2], input_tables[-1] - g = load_graph(i_emb_table, FLAGS.emb_dim, FLAGS.knn_metric, FLAGS.timeout, - FLAGS.knn_strict) + g = load_graph( + i_emb_table, + FLAGS.emb_dim, + FLAGS.knn_metric, + FLAGS.timeout, + FLAGS.knn_strict, + ) hitrate_details_table, total_hitrate_table = FLAGS.outputs.split(',') cluster = tf.train.ClusterSpec({ diff --git a/easy_rec/python/tools/pre_check.py b/easy_rec/python/tools/pre_check.py index da7f1923b..34be0332d 100644 --- a/easy_rec/python/tools/pre_check.py +++ b/easy_rec/python/tools/pre_check.py @@ -11,15 +11,18 @@ from easy_rec.python.utils import config_util from easy_rec.python.utils import fg_util from easy_rec.python.utils import io_util -from easy_rec.python.utils.check_utils import check_env_and_input_path -from easy_rec.python.utils.check_utils import check_sequence + +from easy_rec.python.utils.check_utils import ( # NOQA + check_env_and_input_path, check_sequence, +) if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + level=logging.INFO, +) tf.app.flags.DEFINE_string('pipeline_config_path', None, 'Path to pipeline config ' 'file.') @@ -63,7 +66,8 @@ def _get_input_fn(data_config, data_path, task_index=task_index, task_num=worker_num, - check_mode=True) + check_mode=True, + ) input_fn = input_obj.create_input(export_config) return input_fn @@ -93,7 +97,7 @@ def run_check(pipeline_config, input_path): mode=tf.estimator.ModeKeys.EVAL).make_one_shot_iterator() with tf.Session() as sess: try: - while (True): + while True: input_feas, input_lbls = input_iter.get_next() features = sess.run(input_feas) check_sequence(pipeline_config, features) @@ -108,8 +112,8 @@ def main(argv): if FLAGS.data_input_path: input_path = ','.join(FLAGS.data_input_path) else: - assert pipeline_config.train_input_path or pipeline_config.eval_input_path, \ - 'input_path should not be empty when checking!' + assert (pipeline_config.train_input_path or pipeline_config.eval_input_path + ), 'input_path should not be empty when checking!' input_path = pipeline_config.train_input_path + ',' + pipeline_config.eval_input_path run_check(pipeline_config, input_path) diff --git a/easy_rec/python/tools/read_kafka.py b/easy_rec/python/tools/read_kafka.py index 57578b863..60c1d5d65 100644 --- a/easy_rec/python/tools/read_kafka.py +++ b/easy_rec/python/tools/read_kafka.py @@ -29,7 +29,8 @@ consumer = KafkaConsumer( group_id=args.group, bootstrap_servers=servers, - consumer_timeout_ms=args.timeout * 1000) + consumer_timeout_ms=args.timeout * 1000, + ) if args.partitions is not None: partitions = [int(x) for x in args.partitions.split(',')] diff --git a/easy_rec/python/tools/split_model_pai.py b/easy_rec/python/tools/split_model_pai.py index d86791708..c763ce005 100644 --- a/easy_rec/python/tools/split_model_pai.py +++ b/easy_rec/python/tools/split_model_pai.py @@ -158,9 +158,11 @@ def load_meta_graph_def(model_dir): input_tensor = signatures[sig_name].inputs[input_name] for dim in input_tensor.tensor_shape.dim: input_tensor_shape.append(int(dim.size)) - tf.logging.info('"%s": %s; %s' % - (input_name, _TYPE_TO_STRING[input_tensor.dtype], - input_tensor_shape)) + tf.logging.info('"%s": %s; %s' % ( + input_name, + _TYPE_TO_STRING[input_tensor.dtype], + input_tensor_shape, + )) input_tensor_names[input_name] = input_tensor.name tf.logging.info('[Signature] outputs:') for output_name in signatures[sig_name].outputs: @@ -168,16 +170,25 @@ def load_meta_graph_def(model_dir): output_tensor = signatures[sig_name].outputs[output_name] for dim in output_tensor.tensor_shape.dim: output_tensor_shape.append(int(dim.size)) - tf.logging.info('"%s": %s; %s' % - (output_name, _TYPE_TO_STRING[output_tensor.dtype], - output_tensor_shape)) + tf.logging.info('"%s": %s; %s' % ( + output_name, + _TYPE_TO_STRING[output_tensor.dtype], + output_tensor_shape, + )) output_tensor_names[output_name] = output_tensor.name return meta_graph_def, variable_protos, input_tensor_names, output_tensor_names -def export(model_dir, meta_graph_def, variable_protos, input_tensor_names, - output_tensor_names, part_name, part_dir): +def export( + model_dir, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + part_name, + part_dir, +): """Export subpart saved model. Args: @@ -228,19 +239,20 @@ def export(model_dir, meta_graph_def, variable_protos, input_tensor_names, graph.get_tensor_by_name(output_tensor_names[output_name])) signature_outputs[output_name] = tensor_info - prediction_signature = ( - tf.saved_model.signature_def_utils.build_signature_def( - inputs=signature_inputs, - outputs=signature_outputs, - method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME - )) + prediction_signature = tf.saved_model.signature_def_utils.build_signature_def( + inputs=signature_inputs, + outputs=signature_outputs, + method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME, + ) builder.add_meta_graph_and_variables( - sess, [tf.saved_model.tag_constants.SERVING], + sess, + [tf.saved_model.tag_constants.SERVING], signature_def_map={ signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: prediction_signature, - }) + }, + ) builder.save() config_path = os.path.join(model_dir, 'assets/pipeline.config') assert tf.gfile.Exists(config_path) @@ -259,8 +271,12 @@ def export(model_dir, meta_graph_def, variable_protos, input_tensor_names, def main(argv): model_dir = search_pb(FLAGS.model_dir) tf.logging.info('Loading meta graph...') - meta_graph_def, variable_protos, input_tensor_names, output_tensor_names = load_meta_graph_def( - model_dir) + ( + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + ) = load_meta_graph_def(model_dir) tf.logging.info('Exporting user part model...') export( model_dir, @@ -269,7 +285,8 @@ def main(argv): input_tensor_names, output_tensor_names, part_name='user', - part_dir=FLAGS.user_model_dir) + part_dir=FLAGS.user_model_dir, + ) tf.logging.info('Exporting item part model...') export( model_dir, @@ -278,7 +295,8 @@ def main(argv): input_tensor_names, output_tensor_names, part_name='item', - part_dir=FLAGS.item_model_dir) + part_dir=FLAGS.item_model_dir, + ) if __name__ == '__main__': diff --git a/easy_rec/python/tools/split_pdn_model_pai.py b/easy_rec/python/tools/split_pdn_model_pai.py index 78932c297..5cd414a75 100644 --- a/easy_rec/python/tools/split_pdn_model_pai.py +++ b/easy_rec/python/tools/split_pdn_model_pai.py @@ -150,9 +150,11 @@ def load_meta_graph_def(model_dir): input_tensor = signatures[sig_name].inputs[input_name] for dim in input_tensor.tensor_shape.dim: input_tensor_shape.append(int(dim.size)) - tf.logging.info('"%s": %s; %s' % - (input_name, _TYPE_TO_STRING[input_tensor.dtype], - input_tensor_shape)) + tf.logging.info('"%s": %s; %s' % ( + input_name, + _TYPE_TO_STRING[input_tensor.dtype], + input_tensor_shape, + )) input_tensor_names[input_name] = input_tensor.name tf.logging.info('[Signature] outputs:') for output_name in signatures[sig_name].outputs: @@ -160,16 +162,25 @@ def load_meta_graph_def(model_dir): output_tensor = signatures[sig_name].outputs[output_name] for dim in output_tensor.tensor_shape.dim: output_tensor_shape.append(int(dim.size)) - tf.logging.info('"%s": %s; %s' % - (output_name, _TYPE_TO_STRING[output_tensor.dtype], - output_tensor_shape)) + tf.logging.info('"%s": %s; %s' % ( + output_name, + _TYPE_TO_STRING[output_tensor.dtype], + output_tensor_shape, + )) output_tensor_names[output_name] = output_tensor.name return meta_graph_def, variable_protos, input_tensor_names, output_tensor_names -def export(model_dir, meta_graph_def, variable_protos, input_tensor_names, - output_tensor_names, part_name, part_dir): +def export( + model_dir, + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + part_name, + part_dir, +): """Export subpart saved model. Args: @@ -220,19 +231,20 @@ def export(model_dir, meta_graph_def, variable_protos, input_tensor_names, graph.get_tensor_by_name(output_tensor_names[output_name])) signature_outputs[output_name] = tensor_info - prediction_signature = ( - tf.saved_model.signature_def_utils.build_signature_def( - inputs=signature_inputs, - outputs=signature_outputs, - method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME - )) + prediction_signature = tf.saved_model.signature_def_utils.build_signature_def( + inputs=signature_inputs, + outputs=signature_outputs, + method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME, + ) builder.add_meta_graph_and_variables( - sess, [tf.saved_model.tag_constants.SERVING], + sess, + [tf.saved_model.tag_constants.SERVING], signature_def_map={ signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: prediction_signature, - }) + }, + ) builder.save() config_path = os.path.join(model_dir, 'assets/pipeline.config') assert tf.gfile.Exists(config_path) @@ -245,8 +257,12 @@ def export(model_dir, meta_graph_def, variable_protos, input_tensor_names, def main(argv): model_dir = search_pb(FLAGS.model_dir) tf.logging.info('Loading meta graph...') - meta_graph_def, variable_protos, input_tensor_names, output_tensor_names = load_meta_graph_def( - model_dir) + ( + meta_graph_def, + variable_protos, + input_tensor_names, + output_tensor_names, + ) = load_meta_graph_def(model_dir) tf.logging.info('Exporting trigger part model...') export( model_dir, @@ -255,7 +271,8 @@ def main(argv): input_tensor_names, output_tensor_names, part_name='trigger_out', - part_dir=FLAGS.trigger_model_dir) + part_dir=FLAGS.trigger_model_dir, + ) tf.logging.info('Exporting sim part model...') export( model_dir, @@ -264,7 +281,8 @@ def main(argv): input_tensor_names, output_tensor_names, part_name='sim_out', - part_dir=FLAGS.sim_model_dir) + part_dir=FLAGS.sim_model_dir, + ) if __name__ == '__main__': diff --git a/easy_rec/python/tools/view_saved_model.py b/easy_rec/python/tools/view_saved_model.py index 022bcf1aa..7b11589f2 100644 --- a/easy_rec/python/tools/view_saved_model.py +++ b/easy_rec/python/tools/view_saved_model.py @@ -9,7 +9,8 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + level=logging.INFO, +) if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/easy_rec/python/tools/write_kafka.py b/easy_rec/python/tools/write_kafka.py index 5dfa7dfd2..37902de02 100644 --- a/easy_rec/python/tools/write_kafka.py +++ b/easy_rec/python/tools/write_kafka.py @@ -43,16 +43,19 @@ name=args.topic, num_partitions=1, replication_factor=1, - topic_configs={'max.message.bytes': 1024 * 1024 * 1024}) + topic_configs={'max.message.bytes': 1024 * 1024 * 1024}, + ) ], - validate_only=False) + validate_only=False, + ) logging.info('create increment save topic: %s' % args.topic) admin_clt.close() producer = KafkaProducer( bootstrap_servers=servers, request_timeout_ms=args.timeout * 1000, - api_version=(0, 10, 1)) + api_version=(0, 10, 1), + ) i = 1 with open(args.input_path, 'r') as fin: diff --git a/easy_rec/python/train_eval.py b/easy_rec/python/train_eval.py index bafdf0c1a..84c5a08b9 100644 --- a/easy_rec/python/train_eval.py +++ b/easy_rec/python/train_eval.py @@ -9,28 +9,29 @@ from easy_rec.python.main import _train_and_evaluate_impl from easy_rec.python.protos.train_pb2 import DistributionStrategy -from easy_rec.python.utils import config_util -from easy_rec.python.utils import ds_util -from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils import fg_util -from easy_rec.python.utils import hpo_util -from easy_rec.python.utils.config_util import process_neg_sampler_data_path -from easy_rec.python.utils.config_util import set_eval_input_path -from easy_rec.python.utils.config_util import set_train_input_path + +from easy_rec.python.utils import ( # NOQA + config_util, ds_util, estimator_utils, fg_util, hpo_util, +) +from easy_rec.python.utils.config_util import ( # NOQA + process_neg_sampler_data_path, set_eval_input_path, set_train_input_path, +) if tf.__version__.startswith('1.'): from tensorflow.python.platform import gfile else: import tensorflow.io.gfile as gfile -from easy_rec.python.utils.distribution_utils import set_tf_config_and_get_train_worker_num_on_ds # NOQA +from easy_rec.python.utils.distribution_utils import ( # NOQA + set_tf_config_and_get_train_worker_num_on_ds,) if tf.__version__ >= '2.0': tf = tf.compat.v1 logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d : %(message)s', - level=logging.INFO) + level=logging.INFO, +) if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -38,12 +39,14 @@ '--pipeline_config_path', type=str, default=None, - help='Path to pipeline config file.') + help='Path to pipeline config file.', + ) parser.add_argument( '--continue_train', action='store_true', default=False, - help='continue train using existing model_dir') + help='continue train using existing model_dir', + ) parser.add_argument( '--hpo_param_path', type=str, @@ -53,29 +56,34 @@ '--hpo_metric_save_path', type=str, default=None, - help='hyperparameter save metric path') + help='hyperparameter save metric path', + ) parser.add_argument( '--model_dir', type=str, default=None, - help='will update the model_dir in pipeline_config') + help='will update the model_dir in pipeline_config', + ) parser.add_argument( '--train_input_path', type=str, nargs='*', default=None, - help='train data input path') + help='train data input path', + ) parser.add_argument( '--eval_input_path', type=str, nargs='*', default=None, - help='eval data input path') + help='eval data input path', + ) parser.add_argument( '--fit_on_eval', action='store_true', default=False, - help='Fit evaluation data after fitting and evaluating train data') + help='Fit evaluation data after fitting and evaluating train data', + ) parser.add_argument( '--fit_on_eval_steps', type=int, @@ -85,19 +93,20 @@ '--fine_tune_checkpoint', type=str, default=None, - help='will update the train_config.fine_tune_checkpoint in pipeline_config' + help='will update the train_config.fine_tune_checkpoint in pipeline_config', ) parser.add_argument( '--edit_config_json', type=str, default=None, help='edit pipeline config str, example: {"model_dir":"experiments/",' - '"feature_config.feature[0].boundaries":[4,5,6,7]}') + '"feature_config.feature[0].boundaries":[4,5,6,7]}', + ) parser.add_argument( '--ignore_finetune_ckpt_error', action='store_true', default=False, - help='During incremental training, ignore the problem of missing fine_tune_checkpoint files' + help='During incremental training, ignore the problem of missing fine_tune_checkpoint files', ) parser.add_argument( '--odps_config', type=str, default=None, help='odps config path') @@ -172,7 +181,7 @@ estimator_utils.init_hvd() elif pipeline_config.train_config.train_distribute in [ DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy + DistributionStrategy.SokStrategy, ]: estimator_utils.init_hvd() estimator_utils.init_sok() @@ -188,7 +197,8 @@ hpo_util.save_eval_metrics( pipeline_config.model_dir, metric_save_path=args.hpo_metric_save_path, - has_evaluator=False) + has_evaluator=False, + ) else: config_util.auto_expand_share_feature_configs(pipeline_config) _train_and_evaluate_impl( @@ -196,6 +206,7 @@ args.continue_train, args.check_mode, fit_on_eval=args.fit_on_eval, - fit_on_eval_steps=args.fit_on_eval_steps) + fit_on_eval_steps=args.fit_on_eval_steps, + ) else: raise ValueError('pipeline_config_path should not be empty when training!') diff --git a/easy_rec/python/utils/activation.py b/easy_rec/python/utils/activation.py index 89044f7a3..9e3b14516 100644 --- a/easy_rec/python/utils/activation.py +++ b/easy_rec/python/utils/activation.py @@ -30,14 +30,16 @@ def dice(_x, axis=-1, epsilon=1e-9, name='dice', training=True): 'alpha_' + name, _x.get_shape()[-1], initializer=tf.constant_initializer(0.0), - dtype=tf.float32) + dtype=tf.float32, + ) inputs_normed = tf.layers.batch_normalization( inputs=_x, axis=axis, epsilon=epsilon, center=False, scale=False, - training=training) + training=training, + ) x_p = tf.sigmoid(inputs_normed) return alphas * (1.0 - x_p) * _x + x_p * _x diff --git a/easy_rec/python/utils/check_utils.py b/easy_rec/python/utils/check_utils.py index 5a7551745..eea7aeee6 100644 --- a/easy_rec/python/utils/check_utils.py +++ b/easy_rec/python/utils/check_utils.py @@ -16,13 +16,15 @@ def check_split(line, sep, requried_field_num, field_name=''): for one_line in line: field_num = len(one_line.split(sep)) if field_name: - assert_info = 'sep[%s] maybe invalid. field_num=%d, required_num=%d, field: %s, value: %s, ' \ - 'please check separator and data.' % \ - (sep, field_num, requried_field_num, field_name, one_line) + assert_info = ( + 'sep[%s] maybe invalid. field_num=%d, required_num=%d, field: %s, value: %s, ' + 'please check separator and data.' % + (sep, field_num, requried_field_num, field_name, one_line)) else: - assert_info = 'sep[%s] maybe invalid. field_num=%d, required_num=%d, current line is: %s, ' \ - 'please check separator and data.' % \ - (sep, field_num, requried_field_num, one_line) + assert_info = ( + 'sep[%s] maybe invalid. field_num=%d, required_num=%d, current line is: %s, ' + 'please check separator and data.' % + (sep, field_num, requried_field_num, one_line)) assert field_num == requried_field_num, assert_info return True @@ -32,8 +34,12 @@ def check_string_to_number(field_vals, field_name): try: float(val) except: # noqa: E722 - assert False, 'StringToNumber ERROR: cannot convert string_to_number, field: %s, value: %s. ' \ - 'please check data.' % (field_name, val) + assert False, ( + 'StringToNumber ERROR: cannot convert string_to_number, field: %s, value: %s. ' + 'please check data.' % ( + field_name, + val, + )) return True @@ -46,16 +52,20 @@ def check_sequence(pipeline_config_path, features): if not seq_att_maps: return for seq_att_map in seq_att_maps: - assert len(seq_att_map.key) == len(seq_att_map.hist_seq), \ - 'The size of hist_seq must equal to the size of key in one seq_att_map.' + assert len(seq_att_map.key) == len( + seq_att_map.hist_seq + ), 'The size of hist_seq must equal to the size of key in one seq_att_map.' size_list = [] for hist_seq in seq_att_map.hist_seq: cur_seq_size = len(features[hist_seq].values) size_list.append(cur_seq_size) hist_seqs = ' '.join(seq_att_map.hist_seq) - assert len(set(size_list)) == 1, \ - 'SequenceFeature Error: The size in [%s] should be consistent. Please check input: [%s].' % \ - (hist_seqs, hist_seqs) + assert len(set(size_list)) == 1, ( + 'SequenceFeature Error: The size in [%s] should be consistent. Please check input: [%s].' + % ( + hist_seqs, + hist_seqs, + )) def check_env_and_input_path(pipeline_config, input_path): @@ -71,8 +81,10 @@ def check_env_and_input_path(pipeline_config, input_path): ] if input_type in ignore_input_list: return True - assert_info = 'Current InputType is %s, InputPath is %s. Please check InputType and InputPath.' % \ - (input_type_name, input_path) + assert_info = 'Current InputType is %s, InputPath is %s. Please check InputType and InputPath.' % ( + input_type_name, + input_path, + ) if input_type_name.startswith('Odps'): # is on pai for path in input_path.split(','): diff --git a/easy_rec/python/utils/config_util.py b/easy_rec/python/utils/config_util.py index 3c6f385e7..bc76f3489 100644 --- a/easy_rec/python/utils/config_util.py +++ b/easy_rec/python/utils/config_util.py @@ -178,10 +178,26 @@ def save_pipeline_config(pipeline_config, def _get_basic_types(): dtypes = [ - bool, int, str, float, - type(u''), np.float16, np.float32, np.float64, np.char, np.byte, np.uint8, - np.int8, np.int16, np.uint16, np.uint32, np.int32, np.uint64, np.int64, - bool, str + bool, + int, + str, + float, + type(''), + np.float16, + np.float32, + np.float64, + np.char, + np.byte, + np.uint8, + np.int8, + np.int16, + np.uint16, + np.uint32, + np.int32, + np.uint64, + np.int64, + bool, + str, ] if six.PY2: dtypes.append(long) # noqa: F821 @@ -269,7 +285,7 @@ def _get_attr(obj, attr, only_last=False): '<=': lambda x, y: x <= y, '<': lambda x, y: x < y, '>': lambda x, y: x > y, - '=': lambda x, y: x == y + '=': lambda x, y: x == y, } cond_key = None cond_val = None @@ -406,7 +422,7 @@ def parse_time(time_data): Return: timestamp: int """ - if isinstance(time_data, str) or isinstance(time_data, type(u'')): + if isinstance(time_data, str) or isinstance(time_data, type('')): if len(time_data) == 17: return int( datetime.datetime.strptime(time_data, @@ -560,7 +576,8 @@ def process_neg_sampler_data_path(pipeline_config): return hive_util = HiveUtils( data_config=pipeline_config.data_config, - hive_config=pipeline_config.hive_train_input) + hive_config=pipeline_config.hive_train_input, + ) sampler_type = pipeline_config.data_config.WhichOneof('sampler') sampler_config = getattr(pipeline_config.data_config, sampler_type) if hasattr(sampler_config, 'input_path'): @@ -584,12 +601,12 @@ def parse_extra_config_param(extra_args, edit_config_json): arg_num = len(extra_args) arg_id = 0 while arg_id < arg_num: - if extra_args[arg_id].startswith('--data_config.') or \ - extra_args[arg_id].startswith('--train_config.') or \ - extra_args[arg_id].startswith('--feature_config.') or \ - extra_args[arg_id].startswith('--model_config.') or \ - extra_args[arg_id].startswith('--export_config.') or \ - extra_args[arg_id].startswith('--eval_config.'): + if (extra_args[arg_id].startswith('--data_config.') or + extra_args[arg_id].startswith('--train_config.') or + extra_args[arg_id].startswith('--feature_config.') or + extra_args[arg_id].startswith('--model_config.') or + extra_args[arg_id].startswith('--export_config.') or + extra_args[arg_id].startswith('--eval_config.')): tmp_arg = extra_args[arg_id][2:] if '=' in tmp_arg: sep_pos = tmp_arg.find('=') @@ -610,7 +627,6 @@ def parse_extra_config_param(extra_args, edit_config_json): def process_multi_file_input_path(sampler_config_input_path): - if '*' in sampler_config_input_path: input_path = ','.join( file_path diff --git a/easy_rec/python/utils/convert_rtp_fg.py b/easy_rec/python/utils/convert_rtp_fg.py index d665fcd74..13fdd503e 100644 --- a/easy_rec/python/utils/convert_rtp_fg.py +++ b/easy_rec/python/utils/convert_rtp_fg.py @@ -9,12 +9,13 @@ from google.protobuf import text_format from easy_rec.python.protos.dataset_pb2 import DatasetConfig -from easy_rec.python.protos.feature_config_pb2 import FeatureConfig -from easy_rec.python.protos.feature_config_pb2 import FeatureGroupConfig -from easy_rec.python.protos.feature_config_pb2 import WideOrDeep from easy_rec.python.protos.pipeline_pb2 import EasyRecConfig from easy_rec.python.utils import config_util +from easy_rec.python.protos.feature_config_pb2 import ( # NOQA + FeatureConfig, FeatureGroupConfig, WideOrDeep, +) + if tf.__version__ >= '2.0': tf = tf.compat.v1 @@ -77,13 +78,15 @@ def _set_hash_bucket(feature, feature_config, input_field): assert False, 'one of hash_bucket_size,vocab_file,vocab_list,num_buckets must be set' -def process_features(feature_type, - feature_name, - feature, - pipeline_config, - embedding_dim, - incol_separator, - is_sequence=False): +def process_features( + feature_type, + feature_name, + feature, + pipeline_config, + embedding_dim, + incol_separator, + is_sequence=False, +): feature_config = FeatureConfig() feature_config.input_names.append(feature_name) feature_config.separator = incol_separator @@ -237,9 +240,14 @@ def load_input_field_and_feature_config(rtp_fg, if 'feature_name' in feature: feature_type = feature['feature_type'] feature_name = feature['feature_name'] - pipeline_config = process_features(feature_type, feature_name, feature, - pipeline_config, embedding_dim, - incol_separator) + pipeline_config = process_features( + feature_type, + feature_name, + feature, + pipeline_config, + embedding_dim, + incol_separator, + ) elif 'sequence_name' in feature: logging.info('Set sequence_features group later.') sequence_name = feature['sequence_name'] @@ -254,7 +262,8 @@ def load_input_field_and_feature_config(rtp_fg, pipeline_config, embedding_dim, incol_separator, - is_sequence=True) + is_sequence=True, + ) except Exception: logging.info('convert feature[%s] exception[%s]' % (str(feature), traceback.format_exc())) @@ -262,19 +271,21 @@ def load_input_field_and_feature_config(rtp_fg, return pipeline_config -def convert_rtp_fg(rtp_fg, - embedding_dim=16, - batch_size=1024, - label_fields=[], - num_steps=10, - model_type='', - separator='\002', - incol_separator='\003', - train_input_path=None, - eval_input_path=None, - selected_cols='', - input_type='OdpsRTPInput', - is_async=False): +def convert_rtp_fg( + rtp_fg, + embedding_dim=16, + batch_size=1024, + label_fields=[], + num_steps=10, + model_type='', + separator='\002', + incol_separator='\003', + train_input_path=None, + eval_input_path=None, + selected_cols='', + input_type='OdpsRTPInput', + is_async=False, +): with tf.gfile.GFile(rtp_fg, 'r') as fin: rtp_fg = json.load(fin) @@ -336,8 +347,10 @@ def convert_rtp_fg(rtp_fg, sync_replicas: %s } - """ % ('adam_optimizer' if not is_async else 'adam_async_optimizer', - 'true' if not is_async else 'false') + """ % ( + 'adam_optimizer' if not is_async else 'adam_async_optimizer', + 'true' if not is_async else 'false', + ) text_format.Merge(train_config_str, pipeline_config) pipeline_config.train_config.num_steps = num_steps @@ -414,7 +427,7 @@ def convert_rtp_fg(rtp_fg, 'i': 'item', 'ctx': 'combo', 'q': 'combo', - 'comb': 'combo' + 'comb': 'combo', } for feature in rtp_features: feature_name = feature['feature_name'].strip() @@ -447,22 +460,23 @@ def convert_rtp_fg(rtp_fg, multi_tower_config_str = ' multi_tower {\n' for group_name in feature_groups: - multi_tower_config_str += """ + multi_tower_config_str += (""" towers { input: "%s" dnn { hidden_units: [256, 192, 128] } } - """ % group_name + """ % group_name) - multi_tower_config_str = multi_tower_config_str + """ + multi_tower_config_str = ( + multi_tower_config_str + """ final_dnn { hidden_units: [192, 128, 64] } l2_regularization: 1e-4 } - """ + """) text_format.Merge(multi_tower_config_str, pipeline_config.model_config) pipeline_config.model_config.embedding_regularization = 1e-5 @@ -489,13 +503,13 @@ def convert_rtp_fg(rtp_fg, esmm_config_str = ' esmm {\n' for group_name in feature_groups: - esmm_config_str += """ + esmm_config_str += (""" groups { input: "%s" dnn { hidden_units: [256, 128, 96, 64] } - }""" % group_name + }""" % group_name) esmm_config_str += """ ctr_tower { @@ -525,7 +539,10 @@ def convert_rtp_fg(rtp_fg, } } l2_regularization: 1e-6 - }""" % (label_fields[0], label_fields[1]) + }""" % ( + label_fields[0], + label_fields[1], + ) text_format.Merge(esmm_config_str, pipeline_config.model_config) pipeline_config.model_config.embedding_regularization = 5e-5 elif model_type == 'dbmtl': @@ -591,22 +608,30 @@ def convert_rtp_fg(rtp_fg, } l2_regularization: 1e-6 } - """ % (label_fields[0], label_fields[1]) + """ % ( + label_fields[0], + label_fields[1], + ) text_format.Merge(dbmtl_config_str, pipeline_config.model_config) pipeline_config.model_config.embedding_regularization = 5e-6 if model_type in ['wide_and_deep', 'deepfm', 'multi_tower']: - text_format.Merge(""" + text_format.Merge( + """ metrics_set { auc {} } - """, pipeline_config.eval_config) + """, + pipeline_config.eval_config, + ) text_format.Merge( """ export_config { multi_placeholder: false } - """, pipeline_config) + """, + pipeline_config, + ) if edit_config_json: for edit_obj in edit_config_json: diff --git a/easy_rec/python/utils/dag.py b/easy_rec/python/utils/dag.py index 8d0d4b094..b9ae24725 100644 --- a/easy_rec/python/utils/dag.py +++ b/easy_rec/python/utils/dag.py @@ -71,7 +71,6 @@ def rename_edges(self, old_task_name, new_task_name, graph=None): if not graph: graph = self.graph for node, edges in graph.items(): - if node == old_task_name: graph[new_task_name] = copy(edges) del graph[old_task_name] diff --git a/easy_rec/python/utils/distribution_utils.py b/easy_rec/python/utils/distribution_utils.py index 22ae9ecf8..1be75b59c 100644 --- a/easy_rec/python/utils/distribution_utils.py +++ b/easy_rec/python/utils/distribution_utils.py @@ -10,24 +10,27 @@ from easy_rec.python.protos.train_pb2 import DistributionStrategy from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils.estimator_utils import chief_to_master -from easy_rec.python.utils.estimator_utils import master_to_chief + +from easy_rec.python.utils.estimator_utils import ( # NOQA + chief_to_master, master_to_chief, +) DistributionStrategyMap = { '': DistributionStrategy.NoStrategy, 'ps': DistributionStrategy.PSStrategy, 'ess': DistributionStrategy.ExascaleStrategy, 'mirrored': DistributionStrategy.MirroredStrategy, - 'collective': DistributionStrategy.CollectiveAllReduceStrategy + 'collective': DistributionStrategy.CollectiveAllReduceStrategy, } def set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy): if distribute_strategy in [ - DistributionStrategy.PSStrategy, DistributionStrategy.MirroredStrategy, + DistributionStrategy.PSStrategy, + DistributionStrategy.MirroredStrategy, DistributionStrategy.CollectiveAllReduceStrategy, - DistributionStrategy.ExascaleStrategy + DistributionStrategy.ExascaleStrategy, ]: pipeline_config.train_config.sync_replicas = False pipeline_config.train_config.train_distribute = distribute_strategy @@ -42,7 +45,8 @@ def set_tf_config_and_get_train_worker_num( task_index, job_name, distribute_strategy=DistributionStrategy.NoStrategy, - eval_method='none'): + eval_method='none', +): logging.info( 'set_tf_config_and_get_train_worker_num: distribute_strategy = %d' % distribute_strategy) @@ -60,9 +64,10 @@ def set_tf_config_and_get_train_worker_num( if distribute_strategy == DistributionStrategy.MirroredStrategy: assert total_worker_num == 1, 'mirrored distribute strategy only need 1 worker' elif distribute_strategy in [ - DistributionStrategy.NoStrategy, DistributionStrategy.PSStrategy, + DistributionStrategy.NoStrategy, + DistributionStrategy.PSStrategy, DistributionStrategy.CollectiveAllReduceStrategy, - DistributionStrategy.ExascaleStrategy + DistributionStrategy.ExascaleStrategy, ]: cluster, task_type, task_index_ = estimator_utils.parse_tf_config() train_worker_num = 0 @@ -83,15 +88,16 @@ def set_tf_config_and_get_train_worker_num( 'task': { 'type': task_type, 'index': task_index_ - } + }, } os.environ['TF_CONFIG'] = json.dumps(tf_config) else: # backward compatibility, if user does not assign one evaluator in # -Dcluster, we use first worker for chief, second for evaluation train_worker_num = total_worker_num - 1 - assert train_worker_num > 0, 'in distribution mode worker num must be greater than 1, ' \ - 'the second worker will be used as evaluator' + assert train_worker_num > 0, ( + 'in distribution mode worker num must be greater than 1, ' + 'the second worker will be used as evaluator') if len(worker_hosts) > 1: cluster = {'chief': [worker_hosts[0]], 'worker': worker_hosts[2:]} if distribute_strategy != DistributionStrategy.NoStrategy: @@ -104,7 +110,7 @@ def set_tf_config_and_get_train_worker_num( 'task': { 'type': job_name, 'index': task_index - } + }, }) elif job_name == 'worker': if task_index == 0: @@ -113,7 +119,7 @@ def set_tf_config_and_get_train_worker_num( 'task': { 'type': 'chief', 'index': 0 - } + }, }) elif task_index == 1: os.environ['TF_CONFIG'] = json.dumps({ @@ -121,7 +127,7 @@ def set_tf_config_and_get_train_worker_num( 'task': { 'type': 'evaluator', 'index': 0 - } + }, }) else: os.environ['TF_CONFIG'] = json.dumps({ @@ -129,7 +135,7 @@ def set_tf_config_and_get_train_worker_num( 'task': { 'type': job_name, 'index': task_index - 2 - } + }, }) else: if 'evaluator' in cluster: @@ -152,7 +158,7 @@ def set_tf_config_and_get_train_worker_num( 'task': { 'type': 'worker', 'index': train_worker_num - 2 - } + }, } else: tf_config = { @@ -160,7 +166,7 @@ def set_tf_config_and_get_train_worker_num( 'task': { 'type': task_type, 'index': task_index_ - } + }, } os.environ['TF_CONFIG'] = json.dumps(tf_config) else: @@ -174,7 +180,7 @@ def set_tf_config_and_get_train_worker_num( 'task': { 'type': job_name, 'index': task_index - } + }, }) else: if task_index == 0: @@ -191,7 +197,7 @@ def set_tf_config_and_get_train_worker_num( 'task': { 'type': 'worker', 'index': task_index - 1 - } + }, }) if eval_method == 'none': # change master to chief, will not evaluate @@ -200,8 +206,7 @@ def set_tf_config_and_get_train_worker_num( # change chief to master, will evaluate on master chief_to_master() else: - assert distribute_strategy == '', 'invalid distribute_strategy %s'\ - % distribute_strategy + assert distribute_strategy == '', 'invalid distribute_strategy %s' % distribute_strategy cluster, task_type, task_index = estimator_utils.parse_tf_config() print('Final TF_CONFIG = %s' % os.environ.get('TF_CONFIG', '')) tf.logging.info('TF_CONFIG %s' % os.environ.get('TF_CONFIG', '')) diff --git a/easy_rec/python/utils/ds_util.py b/easy_rec/python/utils/ds_util.py index 883e7bcee..25bb880a2 100644 --- a/easy_rec/python/utils/ds_util.py +++ b/easy_rec/python/utils/ds_util.py @@ -41,7 +41,7 @@ def cache_ckpt(pipeline_config): meta_files = [x for x in src_files if '.data-' not in x] if estimator_utils.is_ps(): _, _, ps_id = estimator_utils.parse_tf_config() - ps_id = (ps_id % len(data_files)) + ps_id = ps_id % len(data_files) data_files = data_files[ps_id:] + data_files[:ps_id] src_files = meta_files + data_files else: diff --git a/easy_rec/python/utils/embedding_utils.py b/easy_rec/python/utils/embedding_utils.py index 960513801..89f326e4f 100644 --- a/easy_rec/python/utils/embedding_utils.py +++ b/easy_rec/python/utils/embedding_utils.py @@ -59,7 +59,6 @@ def set_embedding_parallel(): def is_embedding_parallel(): - global embedding_parallel return embedding_parallel diff --git a/easy_rec/python/utils/estimator_utils.py b/easy_rec/python/utils/estimator_utils.py index ea15063d1..c761e6cfc 100644 --- a/easy_rec/python/utils/estimator_utils.py +++ b/easy_rec/python/utils/estimator_utils.py @@ -21,18 +21,19 @@ from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.platform import gfile -from tensorflow.python.training import basic_session_run_hooks -from tensorflow.python.training import session_run_hook from tensorflow.python.training.summary_io import SummaryWriterCache -from easy_rec.python.ops.incr_record import get_sparse_indices -from easy_rec.python.ops.incr_record import kv_resource_incr_gather from easy_rec.python.utils import constant from easy_rec.python.utils import embedding_utils from easy_rec.python.utils import shape_utils +from tensorflow.python.training import basic_session_run_hooks, session_run_hook # NOQA from tensorflow.python.training.basic_session_run_hooks import SecondOrStepTimer # NOQA +from easy_rec.python.ops.incr_record import ( # NOQA + get_sparse_indices, kv_resource_incr_gather, +) + try: import horovod.tensorflow as hvd except Exception: @@ -44,7 +45,7 @@ sok = None try: - from kafka import KafkaProducer, KafkaAdminClient + from kafka import KafkaAdminClient, KafkaProducer from kafka.admin import NewTopic except ImportError as ex: logging.warning('kafka-python is not installed: %s' % str(ex)) @@ -98,13 +99,15 @@ def begin(self): dtypes=[tf.float32], shapes=[()], name='exit_counter', - shared_name='exit_counter') + shared_name='exit_counter', + ) self._signal_que = tf.FIFOQueue( capacity=self._num_worker, dtypes=[tf.string], shapes=[()], name='exit_counter_signal', - shared_name='exit_counter_signal') + shared_name='exit_counter_signal', + ) self._enque = self._queue.enqueue(1.0) self._que_size = self._queue.size() self._deque = self._queue.dequeue() @@ -157,6 +160,7 @@ def _check_flag_file(is_chief, flag_file): time.sleep(1) from atexit import register + register( _check_flag_file, is_chief=self._is_chief, flag_file=self._flag_file) logging.info('ExitBarrier passed') @@ -194,13 +198,15 @@ def begin(self): dtypes=[tf.float32], shapes=[()], name='exit_counter', - shared_name='exit_counter') + shared_name='exit_counter', + ) self._signal_que = tf.FIFOQueue( capacity=self._num_worker, dtypes=[tf.string], shapes=[()], name='exit_counter_signal', - shared_name='exit_counter_signal') + shared_name='exit_counter_signal', + ) self._enque = self._queue.enqueue(1.0) self._que_size = self._queue.size() self._deque = self._queue.dequeue() @@ -254,6 +260,7 @@ def _check_flag_file(is_chief, flag_file): time.sleep(1) from atexit import register + register( _check_flag_file, is_chief=self._is_chief, flag_file=self._flag_file) session.run(self.metric_ops) @@ -283,10 +290,7 @@ def before_run(self, run_context): if self._is_chief: return tf.train.SessionRunArgs([tf.train.get_global_step()]) - def after_run( - self, - run_context, # pylint: disable=unused-argument - run_values): + def after_run(self, run_context, run_values): # pylint: disable=unused-argument if self._is_chief: global_step = run_values.results[0] curr_progress = global_step / self._num_steps @@ -307,17 +311,19 @@ def end(self, session): class CheckpointSaverHook(CheckpointSaverHook): """Saves checkpoints every N steps or seconds.""" - def __init__(self, - checkpoint_dir, - save_secs=None, - save_steps=None, - saver=None, - checkpoint_basename='model.ckpt', - scaffold=None, - listeners=None, - write_graph=True, - data_offset_var=None, - increment_save_config=None): + def __init__( + self, + checkpoint_dir, + save_secs=None, + save_steps=None, + saver=None, + checkpoint_basename='model.ckpt', + scaffold=None, + listeners=None, + write_graph=True, + data_offset_var=None, + increment_save_config=None, + ): """Initializes a `CheckpointSaverHook`. Args: @@ -345,7 +351,8 @@ def __init__(self, saver=saver, checkpoint_basename=checkpoint_basename, scaffold=scaffold, - listeners=listeners) + listeners=listeners, + ) self._cuda_profile_start = 0 self._cuda_profile_stop = 0 self._steps_per_run = 1 @@ -376,12 +383,14 @@ def __init__(self, save_steps = increment_save_config.dense_save_steps self._dense_timer = SecondOrStepTimer( every_secs=save_secs if save_secs > 0 else None, - every_steps=save_steps if save_steps > 0 else None) + every_steps=save_steps if save_steps > 0 else None, + ) save_secs = increment_save_config.sparse_save_secs save_steps = increment_save_config.sparse_save_steps self._sparse_timer = SecondOrStepTimer( every_secs=save_secs if save_secs > 0 else None, - every_steps=save_steps if save_steps > 0 else None) + every_steps=save_steps if save_steps > 0 else None, + ) self._dense_timer.update_last_triggered_step(0) self._sparse_timer.update_last_triggered_step(0) @@ -399,8 +408,10 @@ def __init__(self, if 'EmbeddingVariable' in str(type(sparse_var)): self._sparse_values.append( kv_resource_incr_gather( - sparse_var._handle, sparse_indice, - np.zeros(sparse_var.shape.as_list(), dtype=np.float32))) + sparse_var._handle, + sparse_indice, + np.zeros(sparse_var.shape.as_list(), dtype=np.float32), + )) # sparse_var.sparse_read(sparse_indice)) else: self._sparse_values.append( @@ -415,7 +426,8 @@ def __init__(self, admin_clt = KafkaAdminClient( bootstrap_servers=increment_save_config.kafka.server, request_timeout_ms=self._kafka_timeout_ms, - api_version_auto_timeout_ms=self._kafka_timeout_ms) + api_version_auto_timeout_ms=self._kafka_timeout_ms, + ) if self._topic not in admin_clt.list_topics(): admin_clt.create_topics( new_topics=[ @@ -425,9 +437,11 @@ def __init__(self, replication_factor=1, topic_configs={ 'max.message.bytes': self._kafka_max_msg_size - }) + }, + ) ], - validate_only=False) + validate_only=False, + ) logging.info('create increment save topic: %s' % self._topic) admin_clt.close() @@ -436,7 +450,8 @@ def __init__(self, bootstrap_servers=servers, max_request_size=self._kafka_max_req_size, api_version_auto_timeout_ms=self._kafka_timeout_ms, - request_timeout_ms=self._kafka_timeout_ms) + request_timeout_ms=self._kafka_timeout_ms, + ) elif increment_save_config.HasField('fs'): fs = increment_save_config.fs if fs.relative: @@ -464,8 +479,11 @@ def after_create_session(self, session, coord): # We do write graph and saver_def at the first call of before_run. # We cannot do this in begin, since we let other hooks to change graph and # add variables at begin. Graph is finalized after all begin calls. - tf.train.write_graph(tf.get_default_graph().as_graph_def(add_shapes=True), - self._checkpoint_dir, 'graph.pbtxt') + tf.train.write_graph( + tf.get_default_graph().as_graph_def(add_shapes=True), + self._checkpoint_dir, + 'graph.pbtxt', + ) saver_def = self._get_saver().saver_def if self._get_saver() else None graph = tf.get_default_graph() meta_graph_def = meta_graph.create_meta_graph_def( @@ -637,12 +655,14 @@ def _save(self, session, step): session, self._save_path, global_step=step, - write_meta_graph=self._write_graph) + write_meta_graph=self._write_graph, + ) self._summary_writer.add_session_log( tf.SessionLog( status=tf.SessionLog.CHECKPOINT, checkpoint_path=self._save_path), - step) + step, + ) should_stop = False for l in self._listeners: # noqa: E741 @@ -656,12 +676,12 @@ def _save(self, session, step): def end(self, session): global_step = session.run(self._global_step_tensor) super(CheckpointSaverHook, self).end(session) - if self._dense_timer is not None and \ - global_step != self._dense_timer.last_triggered_step(): + if self._dense_timer is not None and global_step != self._dense_timer.last_triggered_step( + ): self._dense_timer.update_last_triggered_step(global_step) self._send_dense(global_step, session) - if self._sparse_timer is not None and \ - global_step != self._sparse_timer.last_triggered_step(): + if self._sparse_timer is not None and global_step != self._sparse_timer.last_triggered_step( + ): self._sparse_timer.update_last_triggered_step(global_step) self._send_sparse(global_step, session) @@ -707,8 +727,8 @@ def begin(self): for var_name in sorted(vars_not_inited.keys()): f.write('%s:%s\n' % (var_name, vars_not_inited[var_name])) assert not has_shape_unmatch, 'exist variable shape not match, restore failed' - assert len(vars_not_inited.keys()) == 0, \ - 'exist variable shape not inited, restore failed' + assert len(vars_not_inited.keys() + ) == 0, 'exist variable shape not inited, restore failed' def after_create_session(self, session, coord): assert self._restore_op is not None @@ -750,6 +770,7 @@ def after_create_session(self, session, coord): class MultipleCheckpointsRestoreHook(SessionRunHook): """Restore variable from numpy checkpoint.""" + SEP = ';' def __init__(self, ckpt_paths): diff --git a/easy_rec/python/utils/expr_util.py b/easy_rec/python/utils/expr_util.py index a236bdfd9..c2178b566 100644 --- a/easy_rec/python/utils/expr_util.py +++ b/easy_rec/python/utils/expr_util.py @@ -7,7 +7,7 @@ def _process_multi_expr(expr): idx = 0 two_expr = ['>=', '<=', '=='] expr_list = [] - while (idx < size): + while idx < size: if idx + 2 <= size and expr[idx:idx + 2] in two_expr: expr_list.append(expr[idx:idx + 2]) idx += 2 diff --git a/easy_rec/python/utils/fg_util.py b/easy_rec/python/utils/fg_util.py index c394444bf..b4aa978dc 100644 --- a/easy_rec/python/utils/fg_util.py +++ b/easy_rec/python/utils/fg_util.py @@ -7,7 +7,8 @@ from easy_rec.python.protos.feature_config_pb2 import FeatureConfig from easy_rec.python.utils.config_util import get_compatible_feature_configs -from easy_rec.python.utils.convert_rtp_fg import load_input_field_and_feature_config # NOQA +from easy_rec.python.utils.convert_rtp_fg import ( # NOQA + load_input_field_and_feature_config,) if tf.__version__ >= '2.0': tf = tf.compat.v1 diff --git a/easy_rec/python/utils/hit_rate_utils.py b/easy_rec/python/utils/hit_rate_utils.py index 7ae313548..a55cfa53c 100644 --- a/easy_rec/python/utils/hit_rate_utils.py +++ b/easy_rec/python/utils/hit_rate_utils.py @@ -30,7 +30,8 @@ def load_graph(i_emb_table, emb_dim, knn_metric, timeout, knn_strict): i_emb_table, node_type='i', decoder=gl.Decoder(attr_types=['float'] * emb_dim, attr_delimiter=','), - option=option) + option=option, + ) return g @@ -116,7 +117,8 @@ def reduce_hitrate(cluster, hits, count, task_index): 'worker_count', shape=(), dtype=tf.int32, - initializer=tf.zeros_initializer()) + initializer=tf.zeros_initializer(), + ) var_hits = tf.get_variable( 'hits', shape=(), @@ -126,12 +128,14 @@ def reduce_hitrate(cluster, hits, count, task_index): 'gt_count', shape=(), dtype=tf.float32, - initializer=tf.zeros_initializer()) + initializer=tf.zeros_initializer(), + ) var_total_hitrate = tf.get_variable( 'total_hitate', shape=(), dtype=tf.float32, - initializer=tf.zeros_initializer()) + initializer=tf.zeros_initializer(), + ) var_hits = tf.assign_add(var_hits, hits, use_locking=True) var_gt_count = tf.assign_add(var_gt_count, count, use_locking=True) @@ -180,7 +184,10 @@ def _to_multi_float_attrs(x, userid): else: arr = [_to_float_attrs(sub_x) for sub_x in x.split('|')] assert len(arr) == num_interests, 'invalid arr len=%d, x=%s, userid=%s' % ( - len(arr), x, userid) + len(arr), + x, + userid, + ) return arr src_ids = np.array([src_items[0] for src_items in gt_record]) @@ -217,4 +224,13 @@ def _make_mask(lens): [-1, num_interests, recall_distances.shape[-1]]) hitrates, bad_cases, bad_dists, hits, gt_count = batch_hitrate( src_ids, recall_ids, recall_distances, gt_items, num_interests, mask) - return hits, gt_count, src_ids, recall_ids, recall_distances, hitrates, bad_cases, bad_dists + return ( + hits, + gt_count, + src_ids, + recall_ids, + recall_distances, + hitrates, + bad_cases, + bad_dists, + ) diff --git a/easy_rec/python/utils/hive_utils.py b/easy_rec/python/utils/hive_utils.py index 250344f13..9b348c9fc 100644 --- a/easy_rec/python/utils/hive_utils.py +++ b/easy_rec/python/utils/hive_utils.py @@ -38,14 +38,15 @@ def gen_sql(self): class HiveUtils(object): """Common IO based interface, could run at local or on data science.""" - def __init__(self, - data_config, - hive_config, - selected_cols='', - record_defaults=[], - task_index=0, - task_num=1): - + def __init__( + self, + data_config, + hive_config, + selected_cols='', + record_defaults=[], + task_index=0, + task_num=1, + ): self._data_config = data_config self._hive_config = hive_config @@ -74,7 +75,8 @@ def _construct_hive_connect(self): host=self._hive_config.host, port=self._hive_config.port, username=self._hive_config.username, - database=self._hive_config.database) + database=self._hive_config.database, + ) return conn def hive_read_line(self, input_path, limit_num=None): @@ -124,8 +126,11 @@ def is_table_or_partition_exist(self, partition_name=None, partition_val=None): if partition_name and partition_val: - sql = 'show partitions %s partition(%s=%s)' % (table_name, partition_name, - partition_val) + sql = 'show partitions %s partition(%s=%s)' % ( + table_name, + partition_name, + partition_val, + ) try: res = self.run_sql(sql) if not res: diff --git a/easy_rec/python/utils/hpo_util.py b/easy_rec/python/utils/hpo_util.py index bebf08476..ac446bf28 100644 --- a/easy_rec/python/utils/hpo_util.py +++ b/easy_rec/python/utils/hpo_util.py @@ -123,7 +123,7 @@ def kill_old_proc(tmp_dir, platform='pai'): # clear easy_rec_hpo yarn jobs yarn_job_file = os.path.join(tmp_dir, 'yarn_job.txt') os.system( - "yarn application -list | awk '{ if ($2 == \"easy_rec_hpo\") print $1 }' > %s" + 'yarn application -list | awk \'{ if ($2 == "easy_rec_hpo") print $1 }\' > %s' % yarn_job_file) yarn_job_arr = [] with open(yarn_job_file, 'r') as fin: diff --git a/easy_rec/python/utils/input_utils.py b/easy_rec/python/utils/input_utils.py index cd8a5b975..427ff9280 100644 --- a/easy_rec/python/utils/input_utils.py +++ b/easy_rec/python/utils/input_utils.py @@ -17,7 +17,7 @@ def get_type_defaults(field_type, default_val=''): DatasetConfig.STRING: '', DatasetConfig.BOOL: False, DatasetConfig.FLOAT: 0.0, - DatasetConfig.DOUBLE: 0.0 + DatasetConfig.DOUBLE: 0.0, } assert field_type in type_defaults, 'invalid type: %s' % field_type if default_val == '': @@ -84,7 +84,7 @@ def np_to_tf_type(np_type): np.float: tf.float32, np.float32: tf.float32, float: tf.float32, - np.double: tf.float64 + np.double: tf.float64, } if np_type in _types_map: return _types_map[np_type] diff --git a/easy_rec/python/utils/io_util.py b/easy_rec/python/utils/io_util.py index 92c9c8a1f..9a5ede73c 100644 --- a/easy_rec/python/utils/io_util.py +++ b/easy_rec/python/utils/io_util.py @@ -4,8 +4,10 @@ isort:skip_file """ + import logging from future import standard_library + standard_library.install_aliases() import os @@ -17,6 +19,7 @@ from six.moves import http_client from six.moves import urllib import json + if six.PY2: from urllib import quote else: @@ -128,8 +131,8 @@ def download_and_uncompress_resource(resource_path, dst_dir=EASY_REC_RES_DIR): create_module_dir(dst_dir) _, basename = os.path.split(resource_path) - if not basename.endswith('.tar.gz') and not basename.endswith('.zip') and \ - not basename.endswith('.py'): + if not basename.endswith('.tar.gz') and not basename.endswith( + '.zip') and not basename.endswith('.py'): raise ValueError('resource %s should be tar.gz or zip or py' % resource_path) @@ -157,7 +160,6 @@ def oss_has_t_mode(target_file): try: with tf.gfile.GFile(test_file, 't') as ofile: ofile.write('a') - pass tf.gfile.Remove(test_file) return True except: # noqa: E722 @@ -197,6 +199,7 @@ def convert_tf_flags_to_argparse(flags): """ import argparse import ast + parser = argparse.ArgumentParser() args = {} @@ -209,8 +212,11 @@ def convert_tf_flags_to_argparse(flags): flag_type = type(default) help_str = flag.help or '' args[flag_name] = [ - False, flag_type, default, help_str, - flag.choices if hasattr(flag, 'choices') else None + False, + flag_type, + default, + help_str, + flag.choices if hasattr(flag, 'choices') else None, ] def str2bool(v): @@ -231,7 +237,8 @@ def str2bool(v): nargs='?', const=True, default=False, - help=help_str) + help=help_str, + ) elif flag_type == str: if choices: parser.add_argument( @@ -239,14 +246,16 @@ def str2bool(v): type=str, choices=choices, default=default, - help=help_str) + help=help_str, + ) elif multi: parser.add_argument( '--' + flag_name, type=str, action='append', default=default, - help=help_str) + help=help_str, + ) else: parser.add_argument( '--' + flag_name, type=str, default=default, help=help_str) @@ -255,7 +264,8 @@ def str2bool(v): '--' + flag_name, type=lambda s: ast.literal_eval(s), default=default, - help=help_str) + help=help_str, + ) elif flag_type in (int, float): parser.add_argument( '--' + flag_name, type=flag_type, default=default, help=help_str) diff --git a/easy_rec/python/utils/load_class.py b/easy_rec/python/utils/load_class.py index 9ac749c76..517af9bcc 100644 --- a/easy_rec/python/utils/load_class.py +++ b/easy_rec/python/utils/load_class.py @@ -142,6 +142,7 @@ def import_pkg(pkg_info, prefix_to_remove=None): __import__(module_path) except Exception as e: import traceback + logging.error(traceback.format_exc()) raise ValueError('import module %s failed: %s' % (module_path, str(e))) @@ -168,9 +169,10 @@ def auto_import(user_path=None): # to make class name starts with easy_rec if parent_dir != '': for idx in range(len(pre_defined_dirs)): - pre_defined_dirs[idx] = (os.path.join(parent_dir, - pre_defined_dirs[idx][0]), - pre_defined_dirs[idx][1]) + pre_defined_dirs[idx] = ( + os.path.join(parent_dir, pre_defined_dirs[idx][0]), + pre_defined_dirs[idx][1], + ) prefix_to_remove = parent_dir + '/' if user_path is not None: @@ -193,9 +195,12 @@ def auto_import(user_path=None): def register_class(class_map, class_name, cls): - assert class_name not in class_map or class_map[class_name] == cls, \ + assert class_name not in class_map or class_map[class_name] == cls, ( 'confilict class %s , %s is already register to be %s' % ( - cls, class_name, str(class_map[class_name])) + cls, + class_name, + str(class_map[class_name]), + )) logging.debug('register class %s' % class_name) class_map[class_name] = cls diff --git a/easy_rec/python/utils/meta_graph_editor.py b/easy_rec/python/utils/meta_graph_editor.py index 9fc75f1fe..2df87f299 100644 --- a/easy_rec/python/utils/meta_graph_editor.py +++ b/easy_rec/python/utils/meta_graph_editor.py @@ -11,32 +11,33 @@ from tensorflow.python.saved_model import signature_constants from tensorflow.python.saved_model.loader_impl import SavedModelLoader -from easy_rec.python.utils import conditional -from easy_rec.python.utils import constant -from easy_rec.python.utils import embedding_utils -from easy_rec.python.utils import proto_util +from easy_rec.python.utils import ( # NOQA + conditional, constant, embedding_utils, proto_util, +) EMBEDDING_INITIALIZERS = 'embedding_initializers' class MetaGraphEditor: - def __init__(self, - lookup_lib_path, - saved_model_dir, - redis_url=None, - redis_passwd=None, - redis_timeout=0, - redis_cache_names=[], - oss_path=None, - oss_endpoint=None, - oss_ak=None, - oss_sk=None, - oss_timeout=0, - meta_graph_def=None, - norm_name_to_ids=None, - incr_update_params=None, - debug_dir=''): + def __init__( + self, + lookup_lib_path, + saved_model_dir, + redis_url=None, + redis_passwd=None, + redis_timeout=0, + redis_cache_names=[], + oss_path=None, + oss_endpoint=None, + oss_ak=None, + oss_sk=None, + oss_timeout=0, + meta_graph_def=None, + norm_name_to_ids=None, + incr_update_params=None, + debug_dir='', + ): self._lookup_op = tf.load_op_library(lookup_lib_path) self._debug_dir = debug_dir self._verbose = debug_dir != '' @@ -49,6 +50,7 @@ def __init__(self, assert meta_graph_def, 'either saved_model_dir or meta_graph_def must be set' tf.reset_default_graph() from tensorflow.python.framework import meta_graph + meta_graph.import_scoped_meta_graph_with_return_elements( meta_graph_def, clear_devices=True) # tf.train.import_meta_graph(meta_graph_def) @@ -163,10 +165,14 @@ def _get_share_embed_name(self, x, embed_names): embed_name_sub = '/'.join(tmp_toks) if tmp_name == embed_name_sub: assert not sel_embed_name, 'confusions encountered: %s %s' % ( - x, ','.join(embed_names)) + x, + ','.join(embed_names), + ) sel_embed_name = embed_name assert sel_embed_name, '%s not find in shared_embeddings: %s' % ( - tmp_name, ','.join(embed_names)) + tmp_name, + ','.join(embed_names), + ) return sel_embed_name def _find_embed_combiners(self, norm_embed_names): @@ -182,7 +188,7 @@ def _find_embed_combiners(self, norm_embed_names): combiner_map = { 'SparseSegmentSum': 'sum', 'SparseSegmentMean': 'mean', - 'SparseSegmentSqrtN': 'sqrtn' + 'SparseSegmentSqrtN': 'sqrtn', } for node in self._meta_graph_def.graph_def.node: if node.op in combiner_map: @@ -275,15 +281,15 @@ def _find_embed_names_and_dims(self, norm_embed_names): embed_is_kv = {} for node in self._meta_graph_def.graph_def.node: if 'embedding_weights' in node.name and node.op in [ - 'VariableV2', 'KvVarHandleOp' + 'VariableV2', + 'KvVarHandleOp', ]: tmp = node.attr['shape'].shape.dim[-1].size tmp2 = 1 for x in node.attr['shape'].shape.dim[:-1]: tmp2 = tmp2 * x.size embed_name, _ = proto_util.get_norm_embed_name(node.name, self._verbose) - assert embed_name is not None,\ - 'fail to get_norm_embed_name(%s)' % node.name + assert embed_name is not None, 'fail to get_norm_embed_name(%s)' % node.name embed_dims[embed_name] = tmp embed_sizes[embed_name] = tmp2 embed_is_kv[embed_name] = 1 if node.op == 'KvVarHandleOp' else 0 @@ -345,8 +351,12 @@ def _get_tensor_by_name(tensor_name): self._embed_combiners = self._find_embed_combiners(values.keys()) # get embedding dimensions - self._embed_names, self._embed_dims, self._embed_sizes, self._embed_is_kv\ - = self._find_embed_names_and_dims(values.keys()) + ( + self._embed_names, + self._embed_dims, + self._embed_sizes, + self._embed_is_kv, + ) = self._find_embed_names_and_dims(values.keys()) if not self._embed_name_to_ids: embed_name_uniq = list(set(self._embed_names)) @@ -365,11 +375,20 @@ def _get_tensor_by_name(tensor_name): # normalized feature names self._feature_names = list(values.keys()) - return lookup_input_indices, lookup_input_values, lookup_input_shapes,\ - lookup_input_weights + return ( + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, + ) - def add_lookup_op(self, lookup_input_indices, lookup_input_values, - lookup_input_shapes, lookup_input_weights): + def add_lookup_op( + self, + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, + ): logging.info('add custom lookup operation to lookup embeddings from redis') self._lookup_outs = [None for i in range(len(lookup_input_values))] for i in range(len(lookup_input_values)): @@ -389,7 +408,8 @@ def add_lookup_op(self, lookup_input_indices, lookup_input_values, embedding_dims=self._embed_dims[i:i_1], embedding_names=self._embed_ids[i:i_1], cache=self._is_cache_from_redis, - version=self._meta_graph_version)[0] + version=self._meta_graph_version, + )[0] meta_graph_def = tf.train.export_meta_graph() @@ -401,8 +421,13 @@ def add_lookup_op(self, lookup_input_indices, lookup_input_values, self._meta_graph_def.graph_def, as_utf8=True)) return meta_graph_def - def add_oss_lookup_op(self, lookup_input_indices, lookup_input_values, - lookup_input_shapes, lookup_input_weights): + def add_oss_lookup_op( + self, + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, + ): logging.info('add custom lookup operation to lookup embeddings from oss') place_on_cpu = os.getenv('place_embedding_on_cpu') place_on_cpu = eval(place_on_cpu) if place_on_cpu else False @@ -445,7 +470,8 @@ def add_oss_lookup_op(self, lookup_input_indices, lookup_input_values, embedding_ids=self._embed_ids, embedding_is_kv=self._embed_is_kv, shared_name='embedding_lookup_res', - name='embedding_lookup_fused/lookup') + name='embedding_lookup_fused/lookup', + ) N = np.max([int(x) for x in self._embed_ids]) + 1 uniq_embed_ids = [x for x in range(N)] @@ -470,7 +496,8 @@ def add_oss_lookup_op(self, lookup_input_indices, lookup_input_values, embedding_is_kv=uniq_embed_is_kvs, N=N, shared_name='embedding_lookup_res', - name='embedding_lookup_fused/init') + name='embedding_lookup_fused/init', + ) ops.add_to_collection(EMBEDDING_INITIALIZERS, lookup_init_op) @@ -480,7 +507,8 @@ def add_oss_lookup_op(self, lookup_input_indices, lookup_input_values, embedding_update = self._lookup_op.embedding_update( message=message_ph, shared_name='embedding_lookup_res', - name='embedding_lookup_fused/embedding_update') + name='embedding_lookup_fused/embedding_update', + ) self._embedding_update_inputs['incr_update/sparse/message'] = message_ph self._embedding_update_outputs[ 'incr_update/sparse/embedding_update'] = embedding_update @@ -539,8 +567,12 @@ def _clear_embedding_in_meta_collect(meta_graph_def, collect_name): kept_ops = [ x for x in meta_graph_def.meta_info_def.stripped_op_list.op if x.name not in [ - 'InitializeKvVariableOp', 'KvResourceGather', 'KvResourceImportV2', - 'KvVarHandleOp', 'KvVarIsInitializedOp', 'ReadKvVariableOp' + 'InitializeKvVariableOp', + 'KvResourceGather', + 'KvResourceImportV2', + 'KvVarHandleOp', + 'KvVarIsInitializedOp', + 'ReadKvVariableOp', ] ] meta_graph_def.meta_info_def.stripped_op_list.ClearField('op') @@ -667,8 +699,8 @@ def clear_save_assign(self): for tid, node in enumerate(self._all_graph_nodes): if not self._all_graph_node_flags[tid]: continue - if node.op == 'Assign' and 'save/Assign' in node.name and \ - 'embedding_weights' in node.input[0]: + if node.op == 'Assign' and 'save/Assign' in node.name and 'embedding_weights' in node.input[ + 0]: drop_save_assigns.append('^' + node.name) self._all_graph_node_flags[tid] = False elif 'embedding_weights/ConcatPartitions/concat' in node.name: @@ -838,14 +870,20 @@ def drop_dependent_nodes(self): def edit_graph(self): # the main entrance - lookup_input_indices, lookup_input_values, lookup_input_shapes,\ - lookup_input_weights = self.find_lookup_inputs() + ( + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, + ) = self.find_lookup_inputs() # add lookup op to the graph - self._meta_graph_def = self.add_lookup_op(lookup_input_indices, - lookup_input_values, - lookup_input_shapes, - lookup_input_weights) + self._meta_graph_def = self.add_lookup_op( + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, + ) self.clear_meta_graph_embeding(self._meta_graph_def) @@ -890,14 +928,20 @@ def edit_graph(self): def edit_graph_for_oss(self): # the main entrance - lookup_input_indices, lookup_input_values, lookup_input_shapes,\ - lookup_input_weights = self.find_lookup_inputs() + ( + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, + ) = self.find_lookup_inputs() # add lookup op to the graph - self._meta_graph_def = self.add_oss_lookup_op(lookup_input_indices, - lookup_input_values, - lookup_input_shapes, - lookup_input_weights) + self._meta_graph_def = self.add_oss_lookup_op( + lookup_input_indices, + lookup_input_values, + lookup_input_shapes, + lookup_input_weights, + ) self.clear_meta_graph_embeding(self._meta_graph_def) diff --git a/easy_rec/python/utils/odps_util.py b/easy_rec/python/utils/odps_util.py index 47a9d9870..de2d25d00 100644 --- a/easy_rec/python/utils/odps_util.py +++ b/easy_rec/python/utils/odps_util.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Common functions used for odps input.""" + from tensorflow.python.framework import dtypes from easy_rec.python.protos.dataset_pb2 import DatasetConfig @@ -11,7 +12,7 @@ def is_type_compatiable(odps_type, input_type): type_map = { 'bigint': DatasetConfig.INT64, 'string': DatasetConfig.STRING, - 'double': DatasetConfig.DOUBLE + 'double': DatasetConfig.DOUBLE, } tmp_type = type_map[odps_type] if tmp_type == input_type: @@ -32,7 +33,7 @@ def odps_type_to_input_type(odps_type): odps_type_map = { 'bigint': DatasetConfig.INT64, 'string': DatasetConfig.STRING, - 'double': DatasetConfig.DOUBLE + 'double': DatasetConfig.DOUBLE, } assert odps_type in odps_type_map, 'only support [bigint, string, double]' input_type = odps_type_map[odps_type] @@ -63,9 +64,13 @@ def check_input_field_and_types(data_config): type_map = {x: y for x, y in zip(selected_cols, selected_types)} for x, y in zip(input_fields, input_field_types): tmp_type = type_map[x] - assert is_type_compatiable(tmp_type, y), \ - 'feature[%s] type error: odps %s is not compatible with input_type %s' % ( - x, tmp_type, DatasetConfig.FieldType.Name(y)) + assert is_type_compatiable(tmp_type, y), ( + 'feature[%s] type error: odps %s is not compatible with input_type %s' + % ( + x, + tmp_type, + DatasetConfig.FieldType.Name(y), + )) def odps_type_2_tf_type(odps_type): diff --git a/easy_rec/python/utils/pai_util.py b/easy_rec/python/utils/pai_util.py index de7ce99aa..538d280e3 100644 --- a/easy_rec/python/utils/pai_util.py +++ b/easy_rec/python/utils/pai_util.py @@ -9,10 +9,10 @@ import tensorflow as tf if sys.version_info.major == 2: - from urllib2 import urlopen, Request, HTTPError + from urllib2 import HTTPError, Request, urlopen else: - from urllib.request import urlopen, Request from urllib.error import HTTPError + from urllib.request import Request, urlopen def is_on_pai(): @@ -54,9 +54,9 @@ def process_config(configs, task_index=0, worker_num=1): """ configs = configs.split(',') if len(configs) > 1: - assert len(configs) == worker_num, \ - 'number of configs must be equal to number of workers,' + \ - ' when number of configs > 1' + assert len(configs) == worker_num, ( + 'number of configs must be equal to number of workers,' + + ' when number of configs > 1') config = configs[task_index] else: config = configs[0] diff --git a/easy_rec/python/utils/proto_util.py b/easy_rec/python/utils/proto_util.py index c96d41a78..c7b3275b3 100644 --- a/easy_rec/python/utils/proto_util.py +++ b/easy_rec/python/utils/proto_util.py @@ -37,8 +37,8 @@ def get_norm_embed_name(name, verbose=False): if verbose: logging.info('norm %s to %s' % (name, tmp_name)) return tmp_name, 0 - if i > 1 and name_toks[i + 1].startswith('part_') and \ - name_toks[i] == 'embedding_weights': + if i > 1 and name_toks[i + 1].startswith( + 'part_') and name_toks[i] == 'embedding_weights': tmp_name = '/'.join(name_toks[:i]) part_id = name_toks[i + 1].replace('part_', '') part_toks = part_id.split(':') @@ -51,8 +51,8 @@ def get_norm_embed_name(name, verbose=False): # input_layer/app_category_embedding/app_category_embedding_weights/SparseReshape # => input_layer/app_category_embedding for i in range(0, len(name_toks) - 1): - if name_toks[i + 1].endswith('_embedding_weights') or \ - '_embedding_weights_' in name_toks[i + 1]: + if name_toks[i + 1].endswith( + '_embedding_weights') or '_embedding_weights_' in name_toks[i + 1]: tmp_name = '/'.join(name_toks[:i + 1]) if verbose: logging.info('norm %s to %s' % (name, tmp_name)) diff --git a/easy_rec/python/utils/shape_utils.py b/easy_rec/python/utils/shape_utils.py index f54521513..d26b434a9 100644 --- a/easy_rec/python/utils/shape_utils.py +++ b/easy_rec/python/utils/shape_utils.py @@ -14,6 +14,7 @@ # limitations under the License. # ============================================================================== """Utils used to manipulate tensor shapes.""" + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -90,8 +91,10 @@ def pad_tensor(t, length): t_d0 = t_shape[0] pad_d0 = tf.expand_dims(length - t_d0, 0) pad_shape = tf.cond( - tf.greater(t_rank, 1), lambda: tf.concat([pad_d0, t_shape[1:]], 0), - lambda: tf.expand_dims(length - t_d0, 0)) + tf.greater(t_rank, 1), + lambda: tf.concat([pad_d0, t_shape[1:]], 0), + lambda: tf.expand_dims(length - t_d0, 0), + ) padded_t = tf.concat([t, tf.zeros(pad_shape, dtype=t.dtype)], 0) if not _is_tensor(length): padded_t = _set_dim_0(padded_t, length) @@ -235,8 +238,10 @@ def check_min_image_dim(min_dim, image_tensor): shape_assert = tf.Assert( tf.logical_and( tf.greater_equal(tf.shape(image_tensor)[1], min_dim), - tf.greater_equal(tf.shape(image_tensor)[2], min_dim)), - ['image size must be >= {} in both height and width.'.format(min_dim)]) + tf.greater_equal(tf.shape(image_tensor)[2], min_dim), + ), + ['image size must be >= {} in both height and width.'.format(min_dim)], + ) with tf.control_dependencies([shape_assert]): return tf.identity(image_tensor) @@ -268,8 +273,8 @@ def assert_shape_equal(shape_a, shape_b): Raises: ValueError: When shapes are both static and unequal. """ - if (all(isinstance(dim, int) for dim in shape_a) and - all(isinstance(dim, int) for dim in shape_b)): + if all(isinstance(dim, int) for dim in shape_a) and all( + isinstance(dim, int) for dim in shape_b): if shape_a != shape_b: raise ValueError('Unequal shapes {}, {}'.format(shape_a, shape_b)) else: @@ -327,7 +332,10 @@ def assert_box_normalized(boxes, maximum_normalized_coordinate=1.1): return tf.Assert( tf.logical_and( tf.less_equal(box_maximum, maximum_normalized_coordinate), - tf.greater_equal(box_minimum, 0)), [boxes]) + tf.greater_equal(box_minimum, 0), + ), + [boxes], + ) def get_shape_list(tensor, expected_rank=None, name=None): @@ -351,7 +359,7 @@ def get_shape_list(tensor, expected_rank=None, name=None): assert_rank(tensor, expected_rank, name) shape = tensor.shape.as_list() non_static_indexes = [] - for (index, dim) in enumerate(shape): + for index, dim in enumerate(shape): if dim is None: non_static_indexes.append(index) if not non_static_indexes: @@ -397,7 +405,9 @@ def truncate(seq_embed, seq_length): [shape[0], limited_len, shape[2]]) seq_length = tf.where( tf.greater(seq_length, limited_len), - tf.ones_like(seq_length) * limited_len, seq_length) + tf.ones_like(seq_length) * limited_len, + seq_length, + ) return seq_embed, seq_length def keep(seq_embed, seq_length): @@ -406,8 +416,11 @@ def keep(seq_embed, seq_length): shape = get_shape_list(seq_emb) max_seq_len = shape[1] - return tf.cond(max_seq_len > limited_len, lambda: truncate(seq_emb, seq_len), - lambda: keep(seq_emb, seq_len)) + return tf.cond( + max_seq_len > limited_len, + lambda: truncate(seq_emb, seq_len), + lambda: keep(seq_emb, seq_len), + ) def pad_or_truncate_sequence(seq_emb, seq_len, fixed_len): diff --git a/easy_rec/python/utils/test_utils.py b/easy_rec/python/utils/test_utils.py index b71249ac8..42349f90d 100644 --- a/easy_rec/python/utils/test_utils.py +++ b/easy_rec/python/utils/test_utils.py @@ -4,7 +4,9 @@ isort:skip_file """ + from future import standard_library + standard_library.install_aliases() import yaml import glob @@ -187,12 +189,14 @@ def _load_config_for_distribute_eval(pipeline_config_path, test_dir): return pipeline_config -def test_datahub_train_eval(pipeline_config_path, - odps_oss_config, - test_dir, - process_pipeline_func=None, - total_steps=50, - post_check_func=None): +def test_datahub_train_eval( + pipeline_config_path, + odps_oss_config, + test_dir, + process_pipeline_func=None, + total_steps=50, + post_check_func=None, +): gpus = get_available_gpus() if len(gpus) > 0: set_gpu_id(gpus[0]) @@ -231,8 +235,7 @@ def test_datahub_train_eval(pipeline_config_path, pipeline_config = process_pipeline_func(pipeline_config) config_util.save_pipeline_config(pipeline_config, test_dir) test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') - train_cmd = 'python -m easy_rec.python.train_eval --pipeline_config_path %s' % \ - test_pipeline_config_path + train_cmd = 'python -m easy_rec.python.train_eval --pipeline_config_path %s' % test_pipeline_config_path proc = run_cmd(train_cmd, '%s/log_%s.txt' % (test_dir, 'master')) proc_wait(proc, timeout=TEST_TIME_OUT) if proc.returncode != 0: @@ -251,16 +254,18 @@ def _Load_config_for_test_eval(pipeline_config_path): return pipeline_config -def test_single_train_eval(pipeline_config_path, - test_dir, - process_pipeline_func=None, - hyperparam_str='', - total_steps=50, - post_check_func=None, - check_mode=False, - fine_tune_checkpoint=None, - extra_cmd_args=None, - timeout=-1): +def test_single_train_eval( + pipeline_config_path, + test_dir, + process_pipeline_func=None, + hyperparam_str='', + total_steps=50, + post_check_func=None, + check_mode=False, + fine_tune_checkpoint=None, + extra_cmd_args=None, + timeout=-1, +): gpus = get_available_gpus() if len(gpus) > 0: set_gpu_id(gpus[0]) @@ -288,7 +293,7 @@ def test_single_train_eval(pipeline_config_path, test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') train_cmd = 'python -m easy_rec.python.train_eval --pipeline_config_path=' + test_pipeline_config_path if hyperparam_str: - train_cmd += ' --edit_config_json=\'%s\'' % hyperparam_str + train_cmd += " --edit_config_json='%s'" % hyperparam_str if fine_tune_checkpoint: train_cmd += ' --fine_tune_checkpoint %s' % fine_tune_checkpoint if check_mode: @@ -348,7 +353,10 @@ def test_single_predict(test_dir, input_path, output_path, saved_model_dir): set_gpu_id(None) predict_cmd = 'python -m easy_rec.python.predict --input_path %s --output_path %s --saved_model_dir %s' % ( - input_path, output_path, saved_model_dir) + input_path, + output_path, + saved_model_dir, + ) proc = run_cmd(predict_cmd, '%s/log_%s.txt' % (test_dir, 'master')) proc_wait(proc, timeout=TEST_TIME_OUT) @@ -362,8 +370,9 @@ def test_feature_selection(pipeline_config): model_dir = pipeline_config.model_dir pipeline_config_path = os.path.join(model_dir, 'pipeline.config') output_dir = os.path.join(model_dir, 'feature_selection') - cmd = 'python -m easy_rec.python.tools.feature_selection --config_path %s ' \ - '--output_dir %s --topk 5 --visualize true' % (pipeline_config_path, output_dir) + cmd = ('python -m easy_rec.python.tools.feature_selection --config_path %s ' + '--output_dir %s --topk 5 --visualize true' % + (pipeline_config_path, output_dir)) proc = run_cmd(cmd, os.path.join(model_dir, 'log_feature_selection.txt')) proc_wait(proc, timeout=TEST_TIME_OUT) if proc.returncode != 0: @@ -372,10 +381,12 @@ def test_feature_selection(pipeline_config): return True -def yaml_replace(train_yaml_path, - pipline_config_path, - test_pipeline_config_path, - test_export_dir=None): +def yaml_replace( + train_yaml_path, + pipline_config_path, + test_pipeline_config_path, + test_export_dir=None, +): with open(train_yaml_path, 'r', encoding='utf-8') as _file: sample = _file.read() x = yaml.load(sample) @@ -393,13 +404,14 @@ def yaml_replace(train_yaml_path, yaml.dump(x, _file) -def test_hdfs_train_eval(pipeline_config_path, - train_yaml_path, - test_dir, - process_pipeline_func=None, - hyperparam_str='', - total_steps=2000): - +def test_hdfs_train_eval( + pipeline_config_path, + train_yaml_path, + test_dir, + process_pipeline_func=None, + hyperparam_str='', + total_steps=2000, +): gpus = get_available_gpus() if len(gpus) > 0: set_gpu_id(gpus[0]) @@ -431,12 +443,13 @@ def test_hdfs_train_eval(pipeline_config_path, return proc.returncode == 0 -def test_hdfs_eval(pipeline_config_path, - eval_yaml_path, - test_dir, - process_pipeline_func=None, - hyperparam_str=''): - +def test_hdfs_eval( + pipeline_config_path, + eval_yaml_path, + test_dir, + process_pipeline_func=None, + hyperparam_str='', +): gpus = get_available_gpus() if len(gpus) > 0: set_gpu_id(gpus[0]) @@ -463,12 +476,13 @@ def test_hdfs_eval(pipeline_config_path, return proc.returncode == 0 -def test_hdfs_export(pipeline_config_path, - export_yaml_path, - test_dir, - process_pipeline_func=None, - hyperparam_str=''): - +def test_hdfs_export( + pipeline_config_path, + export_yaml_path, + test_dir, + process_pipeline_func=None, + hyperparam_str='', +): gpus = get_available_gpus() if len(gpus) > 0: set_gpu_id(gpus[0]) @@ -485,8 +499,12 @@ def test_hdfs_export(pipeline_config_path, config_util.save_pipeline_config(pipeline_config, test_dir) test_pipeline_config_path = os.path.join(test_dir, 'pipeline.config') test_export_path = os.path.join(test_dir, 'export_dir') - yaml_replace(export_yaml_path, pipeline_config_path, - test_pipeline_config_path, test_export_path) + yaml_replace( + export_yaml_path, + pipeline_config_path, + test_pipeline_config_path, + test_export_path, + ) logging.info('test_pipeline_config_path is %s' % test_pipeline_config_path) eval_cmd = 'el_submit -yaml %s' % export_yaml_path proc = subprocess.Popen(eval_cmd.split(), stderr=subprocess.STDOUT) @@ -523,19 +541,23 @@ def _get_ports(num_worker): if 'ports' in os.environ: ports = os.environ['ports'] port_arr = [int(x) for x in ports.split(',')] - assert len(port_arr) >= num_worker, 'not enough ports: %s, required: %d'\ - % (ports, num_worker) + assert len(port_arr) >= num_worker, 'not enough ports: %s, required: %d' % ( + ports, + num_worker, + ) return port_arr[:num_worker] else: return get_ports_base(num_worker) -def _ps_worker_train(pipeline_config_path, - test_dir, - num_worker, - num_evaluator=0, - fit_on_eval=False, - fit_on_eval_steps=None): +def _ps_worker_train( + pipeline_config_path, + test_dir, + num_worker, + num_evaluator=0, + fit_on_eval=False, + fit_on_eval_steps=None, +): gpus = get_available_gpus() # not enough gpus, run on cpu only if len(gpus) < num_worker: @@ -545,7 +567,7 @@ def _ps_worker_train(pipeline_config_path, cluster = { chief_or_master: ['localhost:%d' % ports[0]], 'worker': ['localhost:%d' % ports[i] for i in range(1, num_worker)], - 'ps': ['localhost:%d' % ports[-1]] + 'ps': ['localhost:%d' % ports[-1]], } tf_config = {'cluster': cluster} procs = {} @@ -595,7 +617,7 @@ def _ps_worker_distribute_eval(pipeline_config_path, cluster = { chief_or_master: ['localhost:%d' % ports[0]], 'worker': ['localhost:%d' % ports[i] for i in range(1, num_worker)], - 'ps': ['localhost:%d' % ports[-1]] + 'ps': ['localhost:%d' % ports[-1]], } tf_config = {'cluster': cluster} procs = {} @@ -665,20 +687,25 @@ def _multi_worker_hvd_train(pipeline_config_path, test_dir, num_worker): ports = _get_ports(num_worker) hosts = ','.join(['localhost:%d' % ports[i] for i in range(num_worker)]) train_cmd = 'horovodrun -np %d --hosts %s python -m easy_rec.python.train_eval --pipeline_config_path %s' % ( - num_worker, hosts, pipeline_config_path) + num_worker, + hosts, + pipeline_config_path, + ) proc = run_cmd(train_cmd, '%s/log_hvd.txt' % test_dir) proc_wait(proc, timeout=1200) return proc.returncode == 0 -def test_distributed_train_eval(pipeline_config_path, - test_dir, - total_steps=50, - num_evaluator=0, - edit_config_json=None, - use_hvd=False, - fit_on_eval=False, - num_epoch=0): +def test_distributed_train_eval( + pipeline_config_path, + test_dir, + total_steps=50, + num_evaluator=0, + edit_config_json=None, + use_hvd=False, + fit_on_eval=False, + num_epoch=0, +): logging.info('testing pipeline config %s' % pipeline_config_path) pipeline_config = _load_config_for_test(pipeline_config_path, test_dir, total_steps, num_epoch) @@ -689,10 +716,9 @@ def test_distributed_train_eval(pipeline_config_path, pipeline_config.train_config.sync_replicas = False if pipeline_config.train_config.train_distribute not in [ DistributionStrategy.EmbeddingParallelStrategy, - DistributionStrategy.SokStrategy + DistributionStrategy.SokStrategy, ]: - pipeline_config.train_config.train_distribute =\ - DistributionStrategy.HorovodStrategy + pipeline_config.train_config.train_distribute = DistributionStrategy.HorovodStrategy train_config = pipeline_config.train_config config_util.save_pipeline_config(pipeline_config, test_dir) @@ -711,7 +737,8 @@ def test_distributed_train_eval(pipeline_config_path, num_worker, num_evaluator, fit_on_eval, - fit_on_eval_steps=int(total_steps // 2)) + fit_on_eval_steps=int(total_steps // 2), + ) elif train_config.train_distribute == DistributionStrategy.MultiWorkerMirroredStrategy: num_worker = 2 procs = _multi_worker_mirror_train(test_pipeline_config_path, test_dir, @@ -784,7 +811,7 @@ def test_distribute_eval_test(cur_eval_path, test_dir): } difference_num = 0.00001 for k in single_ret.keys(): - if (abs(single_ret[k] - distribute_ret[k]) > difference_num): + if abs(single_ret[k] - distribute_ret[k]) > difference_num: logging.error( 'distribute_eval difference[%.8f] large than threshold[%.8f]' % (abs(single_ret[k] - distribute_ret[k]), difference_num)) @@ -810,9 +837,13 @@ def test_distributed_eval(pipeline_config_path, try: if train_config.train_distribute == DistributionStrategy.NoStrategy: num_worker = 2 - procs = _ps_worker_distribute_eval(test_pipeline_config_path, - checkpoint_path, test_dir, num_worker, - num_evaluator) + procs = _ps_worker_distribute_eval( + test_pipeline_config_path, + checkpoint_path, + test_dir, + num_worker, + num_evaluator, + ) else: raise NotImplementedError diff --git a/easy_rec/python/utils/tf_utils.py b/easy_rec/python/utils/tf_utils.py index 24f47a94a..389b8ded7 100644 --- a/easy_rec/python/utils/tf_utils.py +++ b/easy_rec/python/utils/tf_utils.py @@ -1,6 +1,7 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. """Common functions used for odps input.""" + import json import os @@ -29,7 +30,7 @@ def get_tf_type(field_type): DatasetConfig.STRING: tf.string, DatasetConfig.BOOL: tf.bool, DatasetConfig.FLOAT: tf.float32, - DatasetConfig.DOUBLE: tf.double + DatasetConfig.DOUBLE: tf.double, } assert field_type in type_map, 'invalid type: %s' % field_type return type_map[field_type] @@ -42,7 +43,7 @@ def get_col_type(tf_type): tf.string: 'STRING', tf.float32: 'FLOAT', tf.double: 'DOUBLE', - tf.bool: 'BOOLEAN' + tf.bool: 'BOOLEAN', } assert tf_type in type_map, 'invalid type: %s' % tf_type return type_map[tf_type] diff --git a/easy_rec/version.py b/easy_rec/version.py index fa13b61b9..80ce168e3 100644 --- a/easy_rec/version.py +++ b/easy_rec/version.py @@ -1,4 +1,4 @@ # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. -__version__ = '0.8.6' +__version__ = '0.8.7' diff --git a/examples/data/amazon_books_data/process_amazon.py b/examples/data/amazon_books_data/process_amazon.py index ba1fa7f4c..7fc05f526 100644 --- a/examples/data/amazon_books_data/process_amazon.py +++ b/examples/data/amazon_books_data/process_amazon.py @@ -11,7 +11,8 @@ header=None, names=title, engine='python', - encoding='ISO-8859-1') + encoding='ISO-8859-1', +) print('Reading test data...') test = pd.read_table( 'AmazonBooksData/book_test.txt', @@ -19,7 +20,8 @@ header=None, names=title, engine='python', - encoding='ISO-8859-1') + encoding='ISO-8859-1', +) print('Start processing train data...') train_set = [] diff --git a/examples/data/movielens_1m/process_ml_1m.py b/examples/data/movielens_1m/process_ml_1m.py index 8fca6292a..be4dfd441 100644 --- a/examples/data/movielens_1m/process_ml_1m.py +++ b/examples/data/movielens_1m/process_ml_1m.py @@ -16,7 +16,8 @@ def process_data(): header=None, names=users_title, engine='python', - encoding='ISO-8859-1') + encoding='ISO-8859-1', + ) users = users.filter(regex='UserID|Gender|Age|JobID|ZipCode') # process the gender and age of user gender_map = {'F': 0, 'M': 1} @@ -34,7 +35,8 @@ def process_data(): header=None, names=movies_title, engine='python', - encoding='ISO-8859-1') + encoding='ISO-8859-1', + ) # split the title and year in Feature:'Title' pattern = re.compile(r'^(.*)\((\d+)\)$') @@ -59,7 +61,8 @@ def process_data(): header=None, names=ratings_title, engine='python', - encoding='ISO-8859-1') + encoding='ISO-8859-1', + ) ratings = ratings.filter(regex='UserID|MovieID|ratings') # ratings of 4 and 5 are viewed as positive samples [label:1] # ratings of 0, 1 and 2 are viewed as negative samples [label:0] diff --git a/examples/match_model/readme.md b/examples/match_model/readme.md index a1e4d6b2e..b3630741a 100644 --- a/examples/match_model/readme.md +++ b/examples/match_model/readme.md @@ -4,7 +4,7 @@ # Amazon Books 数据集 -在此数据集中, 提供了2个模型及其负采样版的demo示例 [DSSM](dssm.md) / [DSSM-Negative-Sample](dssm_negative_sample.md) / [MIND](mind.md) / [MIND-Negative-Sample](mind_negative_sample.md)。更多模型可参考[models](../../docs/source/models/)。 +在此数据集中, 提供了2个模型及其负采样版的demo示例 [DSSM](dssm.md) / [DSSM-Negative-Sample](dssm_negative_sample.md) / [MIND](mind.md) / [MIND-Negative-Sample](mind_negative_sample.md)。更多模型可参考[models](../../docs/source/models/)。 - DSSM diff --git a/examples/rank_model/DeepFM.md b/examples/rank_model/DeepFM.md index c5321d0e8..ca97862df 100644 --- a/examples/rank_model/DeepFM.md +++ b/examples/rank_model/DeepFM.md @@ -46,7 +46,7 @@ model_config:{ 需要两个feature_group: wide group和deep group, **group name不能变** -- deepfm: deepfm相关的参数 +- deepfm: deepfm相关的参数 - dnn: deep part的参数配置 diff --git a/examples/rank_model/wide_and_deep.md b/examples/rank_model/wide_and_deep.md index 2ba30d3df..9144891df 100644 --- a/examples/rank_model/wide_and_deep.md +++ b/examples/rank_model/wide_and_deep.md @@ -51,7 +51,7 @@ model_config:{ 需要两个feature_group: wide group和deep group, **group name不能变** -- wide_and_deep: wide_and_deep 相关的参数 +- wide_and_deep: wide_and_deep 相关的参数 - dnn: deep part的参数配置 diff --git a/examples/readme.md b/examples/readme.md index dc2122d2e..8817aa0b8 100644 --- a/examples/readme.md +++ b/examples/readme.md @@ -74,21 +74,21 @@ sudo docker exec -it bash 下面分别是三种常用数据集的下载和预处理： -- MovieLens-1M (详细见:[data/movielens_1m/](data/movielens_1m/)。 也可跳过预处理，直接通过链接下载处理后的数据集： [movies_train_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/movielens_1m/movies_train_data)、[movies_test_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/movielens_1m/movies_test_data)) +- MovieLens-1M (详细见:[data/movielens_1m/](data/movielens_1m/)。 也可跳过预处理，直接通过链接下载处理后的数据集： [movies_train_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/movielens_1m/movies_train_data)、[movies_test_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/movielens_1m/movies_test_data)) ```bash cd examples/data/movielens_1m sh download_and_process.sh ``` -- Criteo-Research-Kaggle (详细见:[data/criteo/](data/criteo/)。也可跳过预处理，直接通过链接下载处理后的数据集： [criteo_train_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/criteo_kaggle/criteo_train_data)、[criteo_test_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/criteo_kaggle/criteo_test_data)) +- Criteo-Research-Kaggle (详细见:[data/criteo/](data/criteo/)。也可跳过预处理，直接通过链接下载处理后的数据集： [criteo_train_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/criteo_kaggle/criteo_train_data)、[criteo_test_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/criteo_kaggle/criteo_test_data)) ```bash cd examples/data/criteo sh download_and_process.sh ``` -- Amazon Books (详细见:[data/amazon_books_data/](data/amazon_books_data/)。也可跳过预处理，直接通过链接直接下载处理后的数据集： [amazon_train_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/amazon_books/amazon_train_data)、[amazon_test_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/amazon_books/amazon_test_data)、[negative_book_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/amazon_books/negative_book_data)) +- Amazon Books (详细见:[data/amazon_books_data/](data/amazon_books_data/)。也可跳过预处理，直接通过链接直接下载处理后的数据集： [amazon_train_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/amazon_books/amazon_train_data)、[amazon_test_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/amazon_books/amazon_test_data)、[negative_book_data](https://easy-rec.oss-cn-hangzhou.aliyuncs.com/data/amazon_books/negative_book_data)) ```bash cd examples/data/amazon_books_data diff --git a/git-lfs/git_lfs.py b/git-lfs/git_lfs.py index 4eefc73e8..2363193ce 100644 --- a/git-lfs/git_lfs.py +++ b/git-lfs/git_lfs.py @@ -13,7 +13,8 @@ logging.basicConfig( format='[%(levelname)s] %(asctime)s %(filename)s[%(lineno)d] : %(message)s', - level=logging.INFO) + level=logging.INFO, +) try: import oss2 @@ -49,11 +50,12 @@ def load_git_url(): for line_str in fin: line_str = line_str.strip() line_json = json.loads(line_str) - git_bin_url_map[line_json['leaf_path']] = (line_json['sig'], - line_json['remote_path']) + git_bin_url_map[line_json['leaf_path']] = ( + line_json['sig'], + line_json['remote_path'], + ) except Exception as ex: logging.warning('exception: %s' % str(ex)) - pass return git_bin_url_map @@ -64,7 +66,10 @@ def save_git_url(git_bin_url_map): for key in keys: val = git_bin_url_map[key] tmp_str = '{"leaf_path": "%s", "sig": "%s", "remote_path": "%s"}' % ( - key, val[0], val[1]) + key, + val[0], + val[1], + ) fout.write('%s\n' % tmp_str) @@ -118,7 +123,9 @@ def save_git_bin(git_arr): leaf_files.sort() # make sure that leaf_name is in front of leaf_file tmp_str = '{"leaf_name": "%s", "leaf_file": %s}' % ( - leaf_path, json.dumps(leaf_files)) + leaf_path, + json.dumps(leaf_files), + ) fout.write('%s\n' % tmp_str) @@ -424,7 +431,6 @@ def get_yes_no(msg): bin_file_map = load_git_bin() except Exception as ex: logging.warning('load_git_bin exception: %s' % traceback.format_exc(ex)) - pass leaf_dirs = list_leafs(add_path) any_new = False for leaf_path, leaf_files in leaf_dirs: @@ -453,7 +459,6 @@ def get_yes_no(msg): bin_file_map = load_git_bin() except Exception as ex: logging.warning('load_git_bin exception: %s' % traceback.format_exc(ex)) - pass leaf_dirs = list_leafs(del_path) any_update = False for leaf_path, leaf_files in leaf_dirs: @@ -520,8 +525,10 @@ def get_yes_no(msg): elif merge_start in [0, 1, 2]: line_json = json.loads(line_str) if line_json['leaf_path'] in git_objs: - git_bin_url_map[line_json['leaf_path']] = (line_json['sig'], - line_json['remote_path']) + git_bin_url_map[line_json['leaf_path']] = ( + line_json['sig'], + line_json['remote_path'], + ) else: logging.warning('invalid state: merge_start = %d, line_str = %s' % (merge_start, line_str)) diff --git a/pai_jobs/deploy_ext.sh b/pai_jobs/deploy_ext.sh index ff17f1760..6dad904b3 100755 --- a/pai_jobs/deploy_ext.sh +++ b/pai_jobs/deploy_ext.sh @@ -60,7 +60,7 @@ then fi ODPSCMD=`which $ODPSCMD` -if [ $? -ne 0 ] && [ $mode -ne 2 ] +if [[ $? -ne 0 ]] && [[ $mode -ne 2 ]] then echo "$ODPSCMD is not in PATH" exit 1 diff --git a/pai_jobs/run.py b/pai_jobs/run.py index 309ec4e7a..9761e98a7 100644 --- a/pai_jobs/run.py +++ b/pai_jobs/run.py @@ -1,5 +1,8 @@ +#!/usr/bin/env python # -*- encoding:utf-8 -*- # Copyright (c) Alibaba, Inc. and its affiliates. +"""Entry script of pai -name easyrec command.""" + from __future__ import print_function import logging @@ -15,18 +18,18 @@ from easy_rec.python.inference.odps_predictor import ODPSPredictor from easy_rec.python.inference.vector_retrieve import VectorRetrieve from easy_rec.python.tools.pre_check import run_check -from easy_rec.python.utils import config_util -from easy_rec.python.utils import constant -from easy_rec.python.utils import estimator_utils -from easy_rec.python.utils import fg_util -from easy_rec.python.utils import hpo_util -from easy_rec.python.utils import pai_util -from easy_rec.python.utils.distribution_utils import DistributionStrategyMap -from easy_rec.python.utils.distribution_utils import set_distribution_config -os.environ['IS_ON_PAI'] = '1' +from easy_rec.python.main import _train_and_evaluate_impl as train_and_evaluate_impl # NOQA -from easy_rec.python.utils.distribution_utils import set_tf_config_and_get_train_worker_num # NOQA +from easy_rec.python.utils import ( # NOQA + config_util, constant, estimator_utils, fg_util, hpo_util, pai_util, +) +from easy_rec.python.utils.distribution_utils import ( # NOQA + DistributionStrategyMap, set_distribution_config, + set_tf_config_and_get_train_worker_num, +) + +os.environ['IS_ON_PAI'] = '1' os.environ['OENV_MultiWriteThreadsNum'] = '4' os.environ['OENV_MultiCopyThreadsNum'] = '4' @@ -38,8 +41,6 @@ logging.error('failed to import tfio: %s' % str(ex)) tf.disable_eager_execution() -from easy_rec.python.main import _train_and_evaluate_impl as train_and_evaluate_impl # NOQA - logging.basicConfig( level=logging.INFO, format='[%(asctime)s][%(levelname)s] %(message)s') @@ -60,8 +61,11 @@ tf.app.flags.DEFINE_boolean('with_evaluator', False, 'whether a evaluator is necessary') tf.app.flags.DEFINE_string( - 'eval_method', 'none', 'default to none, choices are [none: not evaluate,' + - 'master: evaluate on master, separate: evaluate on a separate task]') + 'eval_method', + 'none', + 'default to none, choices are [none: not evaluate,' + + 'master: evaluate on master, separate: evaluate on a separate task]', +) tf.app.flags.DEFINE_string('distribute_strategy', '', 'training distribute strategy') @@ -83,9 +87,11 @@ tf.app.flags.DEFINE_integer('knn_feature_dims', None, 'number of feature dimensions') tf.app.flags.DEFINE_enum( - 'knn_index_type', 'ivfflat', + 'knn_index_type', + 'ivfflat', ['flat', 'ivfflat', 'ivfpq', 'gpu_flat', 'gpu_ivfflat', 'gpu_ivfpg'], - 'knn index type') + 'knn index type', +) tf.app.flags.DEFINE_string('knn_feature_delimiter', ',', 'delimiter for feature vectors') tf.app.flags.DEFINE_integer('knn_nlist', 5, @@ -93,14 +99,19 @@ tf.app.flags.DEFINE_integer('knn_nprobe', 2, 'number of probe part on each worker') tf.app.flags.DEFINE_integer( - 'knn_compress_dim', 8, - 'number of dimensions after compress for `ivfpq` and `gpu_ivfpq`') + 'knn_compress_dim', + 8, + 'number of dimensions after compress for `ivfpq` and `gpu_ivfpq`', +) # flags used for evaluate & export tf.app.flags.DEFINE_string( - 'checkpoint_path', '', 'checkpoint to be evaluated or exported ' + 'checkpoint_path', + '', + 'checkpoint to be evaluated or exported ' 'if not specified, use the latest checkpoint ' - 'in train_config.model_dir') + 'in train_config.model_dir', +) # flags used for evaluate tf.app.flags.DEFINE_string('eval_result_path', 'eval_result.txt', 'eval result metric file') @@ -125,8 +136,10 @@ 'all_cols', '', 'union of (selected_cols, reserved_cols), separated with , ') tf.app.flags.DEFINE_string( - 'all_col_types', '', - 'column data types, for build record defaults, separated with ,') + 'all_col_types', + '', + 'column data types, for build record defaults, separated with ,', +) tf.app.flags.DEFINE_string( 'selected_cols', '', 'columns to keep from input table, they are separated with ,') @@ -134,8 +147,10 @@ 'reserved_cols', '', 'columns to keep from input table, they are separated with ,') tf.app.flags.DEFINE_string( - 'output_cols', None, - 'output columns, such as: score float. multiple columns are separated by ,') + 'output_cols', + None, + 'output columns, such as: score float. multiple columns are separated by ,', +) tf.app.flags.DEFINE_integer('batch_size', 1024, 'predict batch size') tf.app.flags.DEFINE_string( 'profiling_file', None, @@ -257,9 +272,11 @@ def main(argv): num_gpus_per_worker = FLAGS.num_gpus_per_worker worker_hosts = FLAGS.worker_hosts.split(',') num_worker = len(worker_hosts) - assert FLAGS.distribute_strategy in DistributionStrategyMap, \ + assert FLAGS.distribute_strategy in DistributionStrategyMap, ( 'invalid distribute_strategy [%s], available ones are %s' % ( - FLAGS.distribute_strategy, ','.join(DistributionStrategyMap.keys())) + FLAGS.distribute_strategy, + ','.join(DistributionStrategyMap.keys()), + )) if FLAGS.config: config = pai_util.process_config(FLAGS.config, FLAGS.task_index, @@ -310,7 +327,9 @@ def main(argv): assert len( tables ) >= 2, 'at least 2 tables must be specified, but only[%d]: %s' % ( - len(tables), FLAGS.tables) + len(tables), + FLAGS.tables, + ) if FLAGS.train_tables: pipeline_config.train_input_path = FLAGS.train_tables @@ -342,8 +361,12 @@ def main(argv): if FLAGS.train_tables or FLAGS.tables: # parse selected_cols - set_selected_cols(pipeline_config, FLAGS.selected_cols, FLAGS.all_cols, - FLAGS.all_col_types) + set_selected_cols( + pipeline_config, + FLAGS.selected_cols, + FLAGS.all_cols, + FLAGS.all_col_types, + ) else: pipeline_config.data_config.selected_cols = '' pipeline_config.data_config.selected_col_types = '' @@ -375,20 +398,23 @@ def main(argv): FLAGS.task_index, FLAGS.job_name, distribute_strategy=distribute_strategy, - eval_method=FLAGS.eval_method) + eval_method=FLAGS.eval_method, + ) set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy) logging.info('run.py check_mode: %s .' % FLAGS.check_mode) train_and_evaluate_impl( pipeline_config, continue_train=FLAGS.continue_train, - check_mode=FLAGS.check_mode) + check_mode=FLAGS.check_mode, + ) if FLAGS.hpo_metric_save_path: hpo_util.save_eval_metrics( pipeline_config.model_dir, metric_save_path=FLAGS.hpo_metric_save_path, - has_evaluator=(FLAGS.eval_method == 'separate')) + has_evaluator=(FLAGS.eval_method == 'separate'), + ) elif FLAGS.cmd == 'evaluate': check_param('config') @@ -409,14 +435,19 @@ def main(argv): FLAGS.worker_hosts, FLAGS.task_index, FLAGS.job_name, - eval_method='none') + eval_method='none', + ) set_distribution_config(pipeline_config, num_worker, num_gpus_per_worker, distribute_strategy) if FLAGS.eval_tables or FLAGS.tables: # parse selected_cols - set_selected_cols(pipeline_config, FLAGS.selected_cols, FLAGS.all_cols, - FLAGS.all_col_types) + set_selected_cols( + pipeline_config, + FLAGS.selected_cols, + FLAGS.all_cols, + FLAGS.all_col_types, + ) else: pipeline_config.data_config.selected_cols = '' pipeline_config.data_config.selected_col_types = '' @@ -483,9 +514,10 @@ def main(argv): FLAGS.worker_hosts, FLAGS.task_index, FLAGS.job_name, - eval_method='none') + eval_method='none', + ) - assert len(FLAGS.worker_hosts.split(',')) == 1, 'export only need 1 woker' + assert len(FLAGS.worker_hosts.split(',')) == 1, 'export only need 1 worker' config_util.auto_expand_share_feature_configs(pipeline_config) export_dir = FLAGS.export_dir @@ -497,9 +529,14 @@ def main(argv): extra_params = redis_params extra_params.update(oss_params) - export_out_dir = easy_rec.export(export_dir, pipeline_config, - FLAGS.checkpoint_path, FLAGS.asset_files, - FLAGS.verbose, **extra_params) + export_out_dir = easy_rec.export( + export_dir, + pipeline_config, + FLAGS.checkpoint_path, + FLAGS.asset_files, + FLAGS.verbose, + **extra_params, + ) if FLAGS.export_done_file: flag_file = os.path.join(export_out_dir, FLAGS.export_done_file) logging.info('create export done file: %s' % flag_file) @@ -521,7 +558,8 @@ def main(argv): fg_json_path=FLAGS.fg_json_path, profiling_file=profiling_file, all_cols=FLAGS.all_cols, - all_col_types=FLAGS.all_col_types) + all_col_types=FLAGS.all_col_types, + ) input_table, output_table = FLAGS.tables, FLAGS.outputs logging.info('input_table = %s, output_table = %s' % (input_table, output_table)) @@ -533,7 +571,8 @@ def main(argv): output_cols=FLAGS.output_cols, batch_size=FLAGS.batch_size, slice_id=FLAGS.task_index, - slice_num=worker_num) + slice_num=worker_num, + ) elif FLAGS.cmd == 'export_checkpoint': check_param('export_dir') check_param('config') @@ -542,7 +581,8 @@ def main(argv): FLAGS.worker_hosts, FLAGS.task_index, FLAGS.job_name, - eval_method='none') + eval_method='none', + ) assert len(FLAGS.worker_hosts.split(',')) == 1, 'export only need 1 woker' config_util.auto_expand_share_feature_configs(pipeline_config) easy_rec.export_checkpoint( @@ -550,19 +590,26 @@ def main(argv): export_path=FLAGS.export_dir + '/model', checkpoint_path=FLAGS.checkpoint_path, asset_files=FLAGS.asset_files, - verbose=FLAGS.verbose) + verbose=FLAGS.verbose, + ) elif FLAGS.cmd == 'vector_retrieve': check_param('knn_distance') assert FLAGS.knn_feature_dims is not None, '`knn_feature_dims` should not be None' assert FLAGS.knn_num_neighbours is not None, '`knn_num_neighbours` should not be None' - query_table, doc_table, output_table = FLAGS.query_table, FLAGS.doc_table, FLAGS.outputs + query_table, doc_table, output_table = ( + FLAGS.query_table, + FLAGS.doc_table, + FLAGS.outputs, + ) if not query_table: tables = FLAGS.tables.split(',') assert len( tables ) >= 1, 'at least 1 tables must be specified, but only[%d]: %s' % ( - len(tables), FLAGS.tables) + len(tables), + FLAGS.tables, + ) query_table = tables[0] doc_table = tables[1] if len(tables) > 1 else query_table @@ -577,7 +624,8 @@ def main(argv): index_type=FLAGS.knn_index_type, nlist=FLAGS.knn_nlist, nprobe=FLAGS.knn_nprobe, - m=FLAGS.knn_compress_dim) + m=FLAGS.knn_compress_dim, + ) worker_hosts = FLAGS.worker_hosts.split(',') knn(FLAGS.knn_num_neighbours, FLAGS.task_index, len(worker_hosts)) elif FLAGS.cmd == 'check': diff --git a/setup.py b/setup.py index 26b94f707..674b7282f 100644 --- a/setup.py +++ b/setup.py @@ -61,11 +61,12 @@ def parse_require_file(fpath): include_package_data=True, classifiers=[ 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 3' + 'Programming Language :: Python :: 3', ], tests_require=parse_requirements('requirements/tests.txt'), install_requires=parse_requirements('requirements/runtime.txt'), extras_require={ 'all': parse_requirements('requirements.txt'), 'tests': parse_requirements('requirements/tests.txt'), - }) + }, +)
CPU利用率 QPS AVG RT TP99
优化前 89 33 288 362
优化后 93 226 34 57