rnn

cnn_classification/cnn.py

@@ -5,7 +5,7 @@
 class TextCNN(object):
     """
     A CNN class for sentence classification
-    With a embedding layer + a convolutional, max-pooling and softmax layer
+    With an embedding layer + a convolutional, max-pooling and softmax layer
     """
     def __init__(self, sequence_length, num_classes, vocab_size,
                  embedding_size, filter_sizes, num_filters, l2_reg_lambda=0.0):

cnn_classification/train.py

@@ -76,11 +76,11 @@ def train(x_train, y_train, vocab_processor, x_dev, y_dev):
             # initialize cnn
             cnn = TextCNN(sequence_length=x_train.shape[1],
                           num_classes=y_train.shape[1],
-                          vocab_size= len(vocab_processor.vocabulary_),
+                          vocab_size=len(vocab_processor.vocabulary_),
                           embedding_size=FLAGS.embedding_dim,
-                          filter_sizes= list(map(int, FLAGS.filter_sizes.split(','))),
-                          num_filters= FLAGS.num_filters,
-                          l2_reg_lambda= FLAGS.l2_reg_lambda)
+                          filter_sizes=list(map(int, FLAGS.filter_sizes.split(','))),
+                          num_filters=FLAGS.num_filters,
+                          l2_reg_lambda=FLAGS.l2_reg_lambda)
 
             # define training procedure
             global_step = tf.Variable(0, name='global_step', trainable=False)

.gitignore → rnn_classification/.gitignore

@@ -1,3 +1,9 @@
+*.npy
+runs/
+
+# Created by https://www.gitignore.io/api/python,ipythonnotebook
+
+### Python ###
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -8,6 +14,7 @@ __pycache__/
 
 # Distribution / packaging
 .Python
+env/
 build/
 develop-eggs/
 dist/
@@ -19,12 +26,9 @@ lib64/
 parts/
 sdist/
 var/
-wheels/
-share/python-wheels/
 *.egg-info/
 .installed.cfg
 *.egg
-MANIFEST
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -39,77 +43,27 @@ pip-delete-this-directory.txt
 # Unit test / coverage reports
 htmlcov/
 .tox/
-.nox/
 .coverage
 .coverage.*
 .cache
 nosetests.xml
 coverage.xml
-*.cover
-.hypothesis/
-.pytest_cache/
+*,cover
 
 # Translations
 *.mo
 *.pot
 
 # Django stuff:
 *.log
-local_settings.py
-db.sqlite3
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
 
 # Sphinx documentation
 docs/_build/
 
 # PyBuilder
 target/
 
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
 
-# Pyre type checker
-.pyre/
+### IPythonNotebook ###
+# Temporary data
+.ipynb_checkpoints/

rnn_classification/BaseUtil/BaseModel.py

@@ -0,0 +1,24 @@
+import abc
+
+class BaseModel(object):
+    __metaclsaa__ = abc.ABCMeta
+
+    @abc.abstractmethod
+    def instantiate_weight(self):
+        return
+
+
+    @abc.abstractmethod
+    def inference(self):
+        return
+
+
+    @abc.abstractmethod
+    def loss(self):
+        return
+
+
+    @abc.abstractmethod
+    def train(self):
+        return
+

rnn_classification/rnn.py

@@ -0,0 +1,129 @@
+import tensorflow as tf
+import numpy as np
+from tensorflow.contrib import rnn
+from BaseUtil.BaseModel import BaseModel
+
+
+class RNN(BaseModel):
+    """
+    A RNN class for sentence classification
+    With an embedding layer + Bi-LSTM layer + FC layer + softmax
+    """
+    def __init__(self, sequence_length, num_classes, vocab_size,
+                 embed_size, learning_rate, decay_steps, decay_rate,
+                 hidden_size, is_training, l2_lambda, grad_clip,
+                 initializer=tf.random_normal_initializer(stddev=0.1)):
+        """
+        :param sequence_length:
+        :param num_classes:
+        :param vocab_size:
+        :param embedding_size:
+        :param learning_rate:
+        :param decay_steps:
+        :param decay_rate:
+        :param hidden_size:
+        :param is_training:
+        :param l2_lambda:
+        :param grad_clip:
+        :param initializer:
+        """
+        self.num_classes = num_classes
+        self.learning_rate = learning_rate
+        self.decay_steps = decay_steps
+        self.decay_rate = decay_rate
+        self.sequence_length = sequence_length
+        self.vocab_size = vocab_size
+        self.embed_size = embed_size
+        self.hidden_size = hidden_size
+        self.is_training = is_training
+        self.l2_lambda = l2_lambda
+        self.grad_clip = grad_clip
+        self.initializer = initializer
+
+        # define placeholder
+        self.input_x = tf.placeholder(tf.int32, [None, sequence_length], name='input_x')
+        self.input_y = tf.placeholder(tf.float32, [None, num_classes], name='input_y')
+        self.dropout_keep_prob = tf.placeholder(tf.float32, name='dropout_keep_prob')
+
+        self.global_step = tf.Variable(0, name='global_step', trainable=False)
+        self.epoch_step = tf.Variable(0, name='epoch_step', trainable=False)
+        self.epoch_increment = tf.assign(self.epoch_step, tf.add(self.epoch_step, tf.constant(1)))
+
+        self.instantiate_weight()
+        self.logits = self.inference()
+
+        self.loss_val = self.loss()
+        self.train_op = self.train()
+        self.predictions = tf.argmax(self.logits, axis=1, name='predictions')
+
+        correct_prediction = tf.equal(self.predictions, tf.argmax(self.input_y, 1))
+        self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'), name='accuracy')
+
+
+    def instantiate_weight(self):
+        """define all the weights"""
+        with tf.name_scope('weights'):
+            self.Embedding = tf.get_variable('Embedding',shape=[self.vocab_size,self.embed_size],
+                                             initializer=self.initializer)
+            self.W_projection = tf.get_variable('W_projection', shape=[self.hidden_size * 2, self.num_classes],
+                                                    initializer=self.initializer)
+            self.b_projection = tf.get_variable('b_projection', shape=[self.num_classes])
+
+
+    def inference(self):
+        """
+        1. embedding layer
+		2. Bi-LSTM layer
+		3. concat Bi-LSTM output
+		4. FC(full connected) layer
+		5. softmax layer
+        """
+        # embedding layer
+        with tf.name_scope('embedding'):
+            self.embedded_words = tf.nn.embedding_lookup(self.Embedding, self.input_x)
+
+        # Bi-LSTM layer
+        with tf.name_scope('Bi-LSTM'):
+            lstm_fw_cell = rnn.BasicLSTMCell(self.hidden_size)
+            lstm_bw_cell = rnn.BasicLSTMCell(self.hidden_size)
+
+            if self.dropout_keep_prob is not None:
+                lstm_fw_cell = rnn.DropoutWrapper(lstm_fw_cell, output_keep_prob=self.dropout_keep_prob)
+                lstm_bw_cell = rnn.DropoutWrapper(lstm_bw_cell, output_keep_prob=self.dropout_keep_prob)
+
+            outputs, output_states = tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell, lstm_bw_cell,
+                                                                    self.embedded_words,
+                                                                    dtype=tf.float32)
+            output = tf.concat(outputs, axis=2)
+            output_last = tf.reduce_mean(output, axis=1)
+
+            # FC layer
+            with tf.name_scope('output'):
+                self.score = tf.matmul(output_last, self.W_projection) + self.b_projection
+            return self.score
+
+    def loss(self):
+        # loss
+        with tf.name_scope('loss'):
+            losses = tf.nn.softmax_cross_entropy_with_logits(labels=self.input_y, logits=self.score)
+            data_loss = tf.reduce_mean(losses)
+            l2_loss = tf.add_n([tf.nn.l2_loss(cand_v) for cand_v in tf.trainable_variables()
+                                    if 'bias' not in cand_v.name]) * self.l2_lambda
+            data_loss += l2_loss
+            return data_loss
+
+    def train(self):
+        learning_rate = tf.train.exponential_decay(self.learning_rate, self.global_step,
+                                                   self.decay_steps, self.decay_rate, staircase=True)
+        optimizer = tf.train.AdamOptimizer(learning_rate)
+        grads_and_vars = optimizer.compute_gradients(self.loss_val)
+
+        #for idx, (grad, var) in enumerate(grads_and_vars):
+            #if grad is not None:
+                #grads_and_vars[idx] = (tf.clip_by_global_norm(grad, self.grad_clip), var)
+        grads_and_vars = [(tf.clip_by_norm(grad, self.grad_clip), val) for grad, val in grads_and_vars]
+        #grads_and_vars = [(tf.add(grad, tf.random_normal(tf.shape(grad), stddev=self.config.grad_noise)), val) for grad, val in
+               #gvs]
+        train_op = optimizer.apply_gradients(grads_and_vars, global_step=self.global_step)
+        return train_op
+

rnn_classification/run/1542522367.8184037/text_rnn_checkpoint/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "D:\\NLP\\NLP_projects\\text_classification\\rnn_classification\\run\\1542522367.8184037\\text_rnn_checkpoint\\model-5760"
+all_model_checkpoint_paths: "D:\\NLP\\NLP_projects\\text_classification\\rnn_classification\\run\\1542522367.8184037\\text_rnn_checkpoint\\model-5760"