provide tf estimator interface (#238)

.github/ISSUE_TEMPLATE/bug_report.md

@@ -18,9 +18,9 @@ Steps to reproduce the behavior:
 4. See error
 
 **Operating environment(运行环境):**
- - python version [e.g. 3.4, 3.6]
- - tensorflow version [e.g. 1.4.0, 1.12.0]
- - deepctr version [e.g. 0.7.1,]
+ - python version [e.g. 3.5, 3.7]
+ - tensorflow version [e.g. 1.4.0, 1.15.0, 2.2.0]
+ - deepctr version [e.g. 0.8.0,]
 
 **Additional context**
 Add any other context about the problem here.

.github/ISSUE_TEMPLATE/question.md

@@ -16,5 +16,5 @@ Add any other context about the problem here.
 
 **Operating environment(运行环境):**
  - python version [e.g. 3.6]
- - tensorflow version [e.g. 1.4.0,]
- - deepctr version [e.g. 0.7.1,]
+ - tensorflow version [e.g. 1.4.0, 1.5.0, 2.2.0]
+ - deepctr version [e.g. 0.8.0,]

CONTRIBUTING.md

@@ -0,0 +1,10 @@
+This project is under development and we need developers to participate in.
+
+If you
+
+- familiar with and interested in ctr prediction algorithms
+- familiar with tensorflow 
+- have spare time to learn and develop
+- familiar with git
+
+please send a brief introduction of your background and experience to [email protected], welcome to join us!

README.md

@@ -18,11 +18,17 @@
 <!-- [![Gitter](https://badges.gitter.im/DeepCTR/community.svg)](https://gitter.im/DeepCTR/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) -->
 
 
-DeepCTR is a **Easy-to-use**,**Modular** and **Extendible** package of deep-learning based CTR models along with lots of core components layers which can be used to easily build custom models.It is compatible with **tensorflow 1.4+ and 2.0+**.You can use any complex model with `model.fit()`and `model.predict()` .
+DeepCTR is a **Easy-to-use**,**Modular** and **Extendible** package of deep-learning based CTR models along with lots of core components layers which can be used to easily build custom models.You can use any complex model with `model.fit()`，and `model.predict()` .
 
-Let's [**Get Started!**](https://deepctr-doc.readthedocs.io/en/latest/Quick-Start.html)([Chinese Introduction](https://zhuanlan.zhihu.com/p/53231955))
+- Provide `tf.keras.Model` like interface for **quick experiment**. [example](https://deepctr-doc.readthedocs.io/en/latest/Quick-Start.html#getting-started-4-steps-to-deepctr)
+- Provide  `tensorflow estimator` interface for **large scale data** and **distributed training**. [example](https://deepctr-doc.readthedocs.io/en/latest/Quick-Start.html#getting-started-4-steps-to-deepctr-estimator-with-tfrecord)
+- It is compatible with both `tf 1.x`  and `tf 2.x`.
 
 
+
+
+Let's [**Get Started!**](https://deepctr-doc.readthedocs.io/en/latest/Quick-Start.html)([Chinese Introduction](https://zhuanlan.zhihu.com/p/53231955)) and [welcome to join us!](./CONTRIBUTING.md)
+
 ## Models List
 
 |                 Model                  | Paper                                                                                                                                                           |

deepctr/__init__.py

@@ -1,4 +1,4 @@
 from .utils import check_version
 
-__version__ = '0.7.5'
+__version__ = '0.8.0'
 check_version(__version__)

deepctr/estimator/__init__.py

@@ -0,0 +1 @@
+from .models import *

deepctr/estimator/feature_column.py

@@ -0,0 +1,53 @@
+import tensorflow as tf
+from tensorflow.python.feature_column.feature_column import _EmbeddingColumn
+
+from .utils import LINEAR_SCOPE_NAME, variable_scope, get_collection, get_GraphKeys, input_layer, get_losses
+
+
+def linear_model(features, linear_feature_columns):
+    if tf.__version__ >= '2.0.0':
+        linear_logits = tf.compat.v1.feature_column.linear_model(features, linear_feature_columns)
+    else:
+        linear_logits = tf.feature_column.linear_model(features, linear_feature_columns)
+    return linear_logits
+
+
+def get_linear_logit(features, linear_feature_columns, l2_reg_linear=0):
+    with variable_scope(LINEAR_SCOPE_NAME):
+        if not linear_feature_columns:
+            linear_logits = tf.Variable([[0.0]], name='bias_weights')
+        else:
+
+            linear_logits = linear_model(features, linear_feature_columns)
+
+            if l2_reg_linear > 0:
+                for var in get_collection(get_GraphKeys().TRAINABLE_VARIABLES, LINEAR_SCOPE_NAME)[:-1]:
+                    get_losses().add_loss(tf.nn.l2_loss(var, name=var.name.split(":")[0] + "_l2loss"),
+                                          get_GraphKeys().REGULARIZATION_LOSSES)
+    return linear_logits
+
+
+def input_from_feature_columns(features, feature_columns, l2_reg_embedding=0.0):
+    dense_value_list = []
+    sparse_emb_list = []
+    for feat in feature_columns:
+        if is_embedding(feat):
+            sparse_emb = tf.expand_dims(input_layer(features, [feat]), axis=1)
+            sparse_emb_list.append(sparse_emb)
+            if l2_reg_embedding > 0:
+                get_losses().add_loss(tf.nn.l2_loss(sparse_emb, name=feat.name + "_l2loss"),
+                                      get_GraphKeys().REGULARIZATION_LOSSES)
+
+        else:
+            dense_value_list.append(input_layer(features, [feat]))
+
+    return sparse_emb_list, dense_value_list
+
+
+def is_embedding(feature_column):
+    try:
+        from tensorflow.python.feature_column.feature_column_v2 import EmbeddingColumn
+    except:
+        EmbeddingColumn = _EmbeddingColumn
+    return isinstance(feature_column, (_EmbeddingColumn,EmbeddingColumn))
+

deepctr/estimator/inputs.py

@@ -0,0 +1,55 @@
+import tensorflow as tf
+from ..layers.utils import combined_dnn_input
+
+def input_fn_pandas(df, features, label=None, batch_size=256, num_epochs=1, shuffle=False, queue_capacity=2560,
+                    num_threads=1):
+    """
+
+    :param df:
+    :param features:
+    :param label:
+    :param batch_size:
+    :param num_epochs:
+    :param shuffle:
+    :param queue_capacity:
+    :param num_threads:
+    :return:
+    """
+    if label is not None:
+        y = df[label]
+    else:
+        y = None
+    if tf.__version__ >= "2.0.0":
+        return tf.compat.v1.estimator.inputs.pandas_input_fn(df[features], y, batch_size=batch_size,
+                                                             num_epochs=num_epochs,
+                                                             shuffle=shuffle, queue_capacity=queue_capacity,
+                                                             num_threads=num_threads)
+
+    return tf.estimator.inputs.pandas_input_fn(df[features], y, batch_size=batch_size, num_epochs=num_epochs,
+                                               shuffle=shuffle, queue_capacity=queue_capacity, num_threads=num_threads)
+
+
+def input_fn_tfrecord(filenames, feature_description, label=None, batch_size=256, num_epochs=1, shuffle=False,
+                      num_parallel_calls=10):
+    def _parse_examples(serial_exmp):
+        features = tf.parse_single_example(serial_exmp, features=feature_description)
+        if label is not None:
+            labels = features.pop(label)
+            return features, labels
+        return features
+
+    def input_fn():
+        dataset = tf.data.TFRecordDataset(filenames)
+        dataset = dataset.map(_parse_examples, num_parallel_calls=num_parallel_calls).prefetch(
+            buffer_size=batch_size * 10)
+        if shuffle:
+            dataset = dataset.shuffle(buffer_size=batch_size * 10)
+
+        dataset = dataset.repeat(num_epochs).batch(batch_size)
+        iterator = dataset.make_one_shot_iterator()
+
+        return iterator.get_next()
+
+    return input_fn
+
+

deepctr/estimator/models/__init__.py

@@ -0,0 +1,12 @@
+from .afm import AFMEstimator
+from .autoint import AutoIntEstimator
+from .ccpm import CCPMEstimator
+from .dcn import DCNEstimator
+from .deepfm import DeepFMEstimator
+from .fwfm import FwFMEstimator
+from .fibinet import FiBiNETEstimator
+from .fnn import FNNEstimator
+from .nfm import NFMEstimator
+from .pnn import PNNEstimator
+from .wdl import WDLEstimator
+from .xdeepfm import xDeepFMEstimator

deepctr/estimator/models/afm.py

@@ -0,0 +1,67 @@
+# -*- coding:utf-8 -*-
+"""
+
+Author:
+    Weichen Shen,[email protected]
+
+Reference:
+    [1] Xiao J, Ye H, He X, et al. Attentional factorization machines: Learning the weight of feature interactions via attention networks[J]. arXiv preprint arXiv:1708.04617, 2017.
+    (https://arxiv.org/abs/1708.04617)
+
+"""
+import tensorflow as tf
+
+from ..feature_column import get_linear_logit, input_from_feature_columns
+from ..utils import deepctr_model_fn, DNN_SCOPE_NAME, variable_scope
+from ...layers.interaction import AFMLayer, FM
+from ...layers.utils import concat_func
+
+
+def AFMEstimator(linear_feature_columns, dnn_feature_columns, use_attention=True, attention_factor=8,
+                 l2_reg_linear=1e-5, l2_reg_embedding=1e-5, l2_reg_att=1e-5, afm_dropout=0, seed=1024,
+                 task='binary', model_dir=None, config=None, linear_optimizer='Ftrl',
+                 dnn_optimizer='Adagrad'):
+    """Instantiates the Attentional Factorization Machine architecture.
+
+    :param linear_feature_columns: An iterable containing all the features used by linear part of the model.
+    :param dnn_feature_columns: An iterable containing all the features used by deep part of the model.
+    :param use_attention: bool,whether use attention or not,if set to ``False``.it is the same as **standard Factorization Machine**
+    :param attention_factor: positive integer,units in attention net
+    :param l2_reg_linear: float. L2 regularizer strength applied to linear part
+    :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
+    :param l2_reg_att: float. L2 regularizer strength applied to attention net
+    :param afm_dropout: float in [0,1), Fraction of the attention net output units to dropout.
+    :param seed: integer ,to use as random seed.
+    :param task: str, ``"binary"`` for  binary logloss or  ``"regression"`` for regression loss
+    :param model_dir: Directory to save model parameters, graph and etc. This can
+        also be used to load checkpoints from the directory into a estimator
+        to continue training a previously saved model.
+    :param config: tf.RunConfig object to configure the runtime settings.
+    :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to
+        the linear part of the model. Defaults to FTRL optimizer.
+    :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to
+        the deep part of the model. Defaults to Adagrad optimizer.
+    :return: A Tensorflow Estimator  instance.
+
+    """
+
+    def _model_fn(features, labels, mode, config):
+        train_flag = (mode == tf.estimator.ModeKeys.TRAIN)
+
+        linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear)
+
+        with variable_scope(DNN_SCOPE_NAME):
+            sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns,
+                                                                                 l2_reg_embedding=l2_reg_embedding)
+            if use_attention:
+
+                fm_logit = AFMLayer(attention_factor, l2_reg_att, afm_dropout,
+                                    seed)(sparse_embedding_list, training=train_flag)
+            else:
+                fm_logit = FM()(concat_func(sparse_embedding_list, axis=1))
+
+        logits = linear_logits + fm_logit
+
+        return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer)
+
+    return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=config)

deepctr/estimator/models/autoint.py

@@ -0,0 +1,94 @@
+# -*- coding:utf-8 -*-
+"""
+
+Author:
+    Weichen Shen,[email protected]
+
+Reference:
+    [1] Song W, Shi C, Xiao Z, et al. AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks[J]. arXiv preprint arXiv:1810.11921, 2018.(https://arxiv.org/abs/1810.11921)
+
+"""
+
+import tensorflow as tf
+
+from ..feature_column import get_linear_logit, input_from_feature_columns
+from ..utils import deepctr_model_fn, DNN_SCOPE_NAME, variable_scope
+from ...layers.core import DNN
+from ...layers.interaction import InteractingLayer
+from ...layers.utils import concat_func, combined_dnn_input
+
+
+def AutoIntEstimator(linear_feature_columns, dnn_feature_columns, att_layer_num=3, att_embedding_size=8, att_head_num=2,
+                     att_res=True,
+                     dnn_hidden_units=(256, 256), dnn_activation='relu', l2_reg_linear=1e-5,
+                     l2_reg_embedding=1e-5, l2_reg_dnn=0, dnn_use_bn=False, dnn_dropout=0, seed=1024,
+                     task='binary', model_dir=None, config=None, linear_optimizer='Ftrl',
+                     dnn_optimizer='Adagrad'):
+    """Instantiates the AutoInt Network architecture.
+
+    :param linear_feature_columns: An iterable containing all the features used by linear part of the model.
+    :param dnn_feature_columns: An iterable containing all the features used by deep part of the model.
+    :param att_layer_num: int.The InteractingLayer number to be used.
+    :param att_embedding_size: int.The embedding size in multi-head self-attention network.
+    :param att_head_num: int.The head number in multi-head  self-attention network.
+    :param att_res: bool.Whether or not use standard residual connections before output.
+    :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of DNN
+    :param dnn_activation: Activation function to use in DNN
+    :param l2_reg_linear: float. L2 regularizer strength applied to linear part
+    :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
+    :param l2_reg_dnn: float. L2 regularizer strength applied to DNN
+    :param dnn_use_bn:  bool. Whether use BatchNormalization before activation or not in DNN
+    :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate.
+    :param seed: integer ,to use as random seed.
+    :param task: str, ``"binary"`` for  binary logloss or  ``"regression"`` for regression loss
+    :param model_dir: Directory to save model parameters, graph and etc. This can
+        also be used to load checkpoints from the directory into a estimator
+        to continue training a previously saved model.
+    :param config: tf.RunConfig object to configure the runtime settings.
+    :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to
+        the linear part of the model. Defaults to FTRL optimizer.
+    :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to
+        the deep part of the model. Defaults to Adagrad optimizer.
+    :return: A Tensorflow Estimator  instance.
+
+    """
+
+    def _model_fn(features, labels, mode, config):
+        train_flag = (mode == tf.estimator.ModeKeys.TRAIN)
+
+        linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear)
+
+        with variable_scope(DNN_SCOPE_NAME):
+            sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns,
+                                                                                 l2_reg_embedding=l2_reg_embedding)
+            att_input = concat_func(sparse_embedding_list, axis=1)
+
+            for _ in range(att_layer_num):
+                att_input = InteractingLayer(
+                    att_embedding_size, att_head_num, att_res)(att_input)
+            att_output = tf.keras.layers.Flatten()(att_input)
+
+            dnn_input = combined_dnn_input(sparse_embedding_list, dense_value_list)
+
+            if len(dnn_hidden_units) > 0 and att_layer_num > 0:  # Deep & Interacting Layer
+                deep_out = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout,
+                               dnn_use_bn, seed)(dnn_input, training=train_flag)
+                stack_out = tf.keras.layers.Concatenate()([att_output, deep_out])
+                final_logit = tf.keras.layers.Dense(
+                    1, use_bias=False, activation=None)(stack_out)
+            elif len(dnn_hidden_units) > 0:  # Only Deep
+                deep_out = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout,
+                               dnn_use_bn, seed)(dnn_input, training=train_flag)
+                final_logit = tf.keras.layers.Dense(
+                    1, use_bias=False, activation=None)(deep_out)
+            elif att_layer_num > 0:  # Only Interacting Layer
+                final_logit = tf.keras.layers.Dense(
+                    1, use_bias=False, activation=None)(att_output)
+            else:  # Error
+                raise NotImplementedError
+
+        logits = linear_logits + final_logit
+
+        return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer)
+
+    return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=config)