Refactored the code for recommenders and nlp_class2

ann_class2/batch_norm_tf.py

@@ -4,13 +4,15 @@
 # sudo pip install -U future
 
 import numpy as np
-import pandas as pd
+#import pandas as pd
 import matplotlib.pyplot as plt
 import tensorflow as tf
 from sklearn.utils import shuffle
-from sklearn.model_selection import train_test_split
+#from sklearn.model_selection import train_test_split
 from util import get_normalized_data
 
+if tf.__version__.startswith('2'):
+    tf.compat.v1.disable_eager_execution()
 
 def init_weight(M1, M2):
   return np.random.randn(M1, M2) * np.sqrt(2.0 / M1)
@@ -38,13 +40,11 @@ def forward(self, X, is_training, decay=0.9):
     activation = tf.matmul(X, self.W)
     if is_training:
       batch_mean, batch_var = tf.nn.moments(activation, [0])
-      update_running_mean = tf.assign(
-        self.running_mean,
-        self.running_mean * decay + batch_mean * (1 - decay)
+      update_running_mean = self.running_mean.assign(
+      self.running_mean * decay + batch_mean * (1 - decay)
       )
-      update_running_var = tf.assign(
-        self.running_var,
-        self.running_var * decay + batch_var * (1 - decay)
+      update_running_var = self.running_var.assign(
+      self.running_var * decay + batch_var * (1 - decay)
       )
 
       with tf.control_dependencies([update_running_mean, update_running_var]):
@@ -115,8 +115,8 @@ def fit(self, X, Y, Xtest, Ytest, activation=tf.nn.relu, learning_rate=1e-2, epo
     # for train and test (prediction)
 
     # set up theano functions and variables
-    tfX = tf.placeholder(tf.float32, shape=(None, D), name='X')
-    tfY = tf.placeholder(tf.int32, shape=(None,), name='Y')
+    tfX = tf.compat.v1.placeholder(tf.float32, shape=(None, D), name='X')
+    tfY = tf.compat.v1.placeholder(tf.int32, shape=(None,), name='Y')
 
     # for later use
     self.tfX = tfX
@@ -131,7 +131,7 @@ def fit(self, X, Y, Xtest, Ytest, activation=tf.nn.relu, learning_rate=1e-2, epo
     )
     # train_op = tf.train.AdamOptimizer(learning_rate).minimize(cost)
     # train_op = tf.train.RMSPropOptimizer(learning_rate, decay=0.99, momentum=0.9).minimize(cost)
-    train_op = tf.train.MomentumOptimizer(learning_rate, momentum=0.9, use_nesterov=True).minimize(cost)
+    train_op = tf.compat.v1.train.MomentumOptimizer(learning_rate, momentum=0.9, use_nesterov=True).minimize(cost)
     # train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
 
     # for testing
@@ -141,7 +141,7 @@ def fit(self, X, Y, Xtest, Ytest, activation=tf.nn.relu, learning_rate=1e-2, epo
     # accuracy = tf.reduce_mean(1.0*(tfY == tf.argmax(logits, 1)))
 
     # init the variables
-    self.session.run(tf.global_variables_initializer())
+    self.session.run(tf.compat.v1.global_variables_initializer())
 
     n_batches = N // batch_sz
     costs = []
@@ -187,7 +187,7 @@ def main():
 
   ann = ANN([500, 300])
 
-  session = tf.InteractiveSession()
+  session = tf.compat.v1.InteractiveSession()
   ann.set_session(session)
 
   ann.fit(Xtrain, Ytrain, Xtest, Ytest, show_fig=True)

ann_class2/dropout_tensorflow.py

@@ -13,6 +13,8 @@
 from util import get_normalized_data
 from sklearn.utils import shuffle
 
+if tf.__version__.startswith('2'):
+    tf.compat.v1.disable_eager_execution()
 
 class HiddenLayer(object):
     def __init__(self, M1, M2):
@@ -59,8 +61,8 @@ def fit(self, X, Y, Xvalid, Yvalid, lr=1e-4, mu=0.9, decay=0.9, epochs=15, batch
             self.params += h.params
 
         # set up theano functions and variables
-        inputs = tf.placeholder(tf.float32, shape=(None, D), name='inputs')
-        labels = tf.placeholder(tf.int64, shape=(None,), name='labels')
+        inputs = tf.compat.v1.placeholder(tf.float32, shape=(None, D), name='inputs')
+        labels = tf.compat.v1.placeholder(tf.int64, shape=(None,), name='labels')
         logits = self.forward(inputs)
 
         cost = tf.reduce_mean(
@@ -69,7 +71,7 @@ def fit(self, X, Y, Xvalid, Yvalid, lr=1e-4, mu=0.9, decay=0.9, epochs=15, batch
                 labels=labels
             )
         )
-        train_op = tf.train.RMSPropOptimizer(lr, decay=decay, momentum=mu).minimize(cost)
+        train_op = tf.compat.v1.train.RMSPropOptimizer(lr, decay=decay, momentum=mu).minimize(cost)
         # train_op = tf.train.MomentumOptimizer(lr, momentum=mu).minimize(cost)
         # train_op = tf.train.AdamOptimizer(lr).minimize(cost)
         prediction = self.predict(inputs)
@@ -85,8 +87,8 @@ def fit(self, X, Y, Xvalid, Yvalid, lr=1e-4, mu=0.9, decay=0.9, epochs=15, batch
 
         n_batches = N // batch_sz
         costs = []
-        init = tf.global_variables_initializer()
-        with tf.Session() as session:
+        init = tf.compat.v1.global_variables_initializer()
+        with tf.compat.v1.Session() as session:
             session.run(init)
             for i in range(epochs):
                 print("epoch:", i, "n_batches:", n_batches)

ann_class2/tensorflow2.py

@@ -12,11 +12,12 @@
 
 import numpy as np
 import tensorflow as tf
-
 import matplotlib.pyplot as plt
-
 from util import get_normalized_data, y2indicator
 
+if tf.__version__.startswith('2'):
+    tf.compat.v1.disable_eager_execution()
+
 
 def error_rate(p, t):
     return np.mean(p != t)
@@ -31,7 +32,7 @@ def main():
     print_period = 50
 
     lr = 0.00004
-    reg = 0.01
+    #reg = 0.01
 
     Ytrain_ind = y2indicator(Ytrain)
     Ytest_ind = y2indicator(Ytest)
@@ -53,8 +54,8 @@ def main():
 
 
     # define variables and expressions
-    X = tf.placeholder(tf.float32, shape=(None, D), name='X')
-    T = tf.placeholder(tf.float32, shape=(None, K), name='T')
+    X = tf.compat.v1.placeholder(tf.float32, shape=(None, D), name='X')
+    T = tf.compat.v1.placeholder(tf.float32, shape=(None, K), name='T')
     W1 = tf.Variable(W1_init.astype(np.float32))
     b1 = tf.Variable(b1_init.astype(np.float32))
     W2 = tf.Variable(W2_init.astype(np.float32))
@@ -70,19 +71,19 @@ def main():
     # softmax_cross_entropy_with_logits take in the "logits"
     # if you wanted to know the actual output of the neural net,
     # you could pass "Yish" into tf.nn.softmax(logits)
-    cost = tf.reduce_sum(tf.nn.softmax_cross_entropy_with_logits_v2(logits=Yish, labels=T))
+    cost = tf.reduce_sum(tf.nn.softmax_cross_entropy_with_logits(logits=Yish, labels=T))
 
     # we choose the optimizer but don't implement the algorithm ourselves
     # let's go with RMSprop, since we just learned about it.
     # it includes momentum!
-    train_op = tf.train.RMSPropOptimizer(lr, decay=0.99, momentum=0.9).minimize(cost)
+    train_op = tf.compat.v1.train.RMSPropOptimizer(lr, decay=0.99, momentum=0.9).minimize(cost)
 
     # we'll use this to calculate the error rate
     predict_op = tf.argmax(Yish, 1)
 
     costs = []
-    init = tf.global_variables_initializer()
-    with tf.Session() as session:
+    init = tf.compat.v1.global_variables_initializer()
+    with tf.compat.v1.Session() as session:
         session.run(init)
 
         for i in range(max_iter):

ann_class2/util.py

@@ -15,7 +15,7 @@
 import pandas as pd
 import matplotlib.pyplot as plt
 from sklearn.decomposition import PCA
-from sklearn.linear_model import LogisticRegression
+#from sklearn.linear_model import LogisticRegression
 
 
 def get_clouds():

keras_examples/ann.py

@@ -7,8 +7,8 @@
 import matplotlib.pyplot as plt
 
 from util import getKaggleMNIST
-from keras.models import Model
-from keras.layers import Dense, Activation, Input
+from tensorflow.keras.models import Model # type: ignore
+from tensorflow.keras.layers import Dense, Input # type: ignore
 
 
 # get the data
@@ -58,8 +58,8 @@
 plt.show()
 
 # accuracies
-plt.plot(r.history['acc'], label='acc')
-plt.plot(r.history['val_acc'], label='val_acc')
+plt.plot(r.history['accuracy'], label='acc')
+plt.plot(r.history['val_accuracy'], label='val_acc')
 plt.legend()
 plt.show()
 

keras_examples/cnn.py

@@ -5,14 +5,14 @@
 # Note: you may need to update your version of future
 # sudo pip install -U future
 
-from keras.models import Model
-from keras.layers import Dense, Activation, Conv2D, MaxPooling2D, Flatten, Input
+from tensorflow.keras.models import Model # type: ignore
+from tensorflow.keras.layers import Dense, Activation, Conv2D, MaxPooling2D, Flatten, Input # type: ignore
 
 import matplotlib.pyplot as plt
 import pandas as pd
 import numpy as np
 
-from util import getKaggleMNIST3D, getKaggleFashionMNIST3D, getCIFAR10
+from util import getKaggleFashionMNIST3D
 
 
 # get the data
@@ -73,8 +73,8 @@
 plt.show()
 
 # accuracies
-plt.plot(r.history['acc'], label='acc')
-plt.plot(r.history['val_acc'], label='val_acc')
+plt.plot(r.history['accuracy'], label='acc')
+plt.plot(r.history['val_accuracy'], label='val_acc')
 plt.legend()
 plt.show()
 

keras_examples/util.py

@@ -12,12 +12,12 @@
 
 def getKaggleMNIST():
     # https://www.kaggle.com/c/digit-recognizer
-    return getMNISTFormat('../large_files/train.csv')
+    return getMNISTFormat('.\\large_files\\digit-recognizer\\train.csv')
 
 
 def getKaggleFashionMNIST():
     # https://www.kaggle.com/zalando-research/fashionmnist
-    return getMNISTFormat('../large_files/fashionmnist/fashion-mnist_train.csv')
+    return getMNISTFormat('.\\large_files\\fashionmnist\\fashion-mnist_train.csv')
 
 def getMNISTFormat(path):
     # MNIST data:

nlp_class2/bow_classifier.py

@@ -32,7 +32,7 @@ def __init__(self):
     word2vec = {}
     embedding = []
     idx2word = []
-    with open('../large_files/glove.6B/glove.6B.50d.txt') as f:
+    with open('../large_files/glove.6B/glove.6B.50d.txt', encoding='utf-8') as f:
       # is just a space-separated text file in the format:
       # word vec[0] vec[1] vec[2] ...
       for line in f:

nlp_class2/glove.py

@@ -20,7 +20,7 @@
 import sys
 sys.path.append(os.path.abspath('..'))
 from rnn_class.util import get_wikipedia_data
-from rnn_class.brown import get_sentences_with_word2idx_limit_vocab, get_sentences_with_word2idx
+from rnn_class.brown import get_sentences_with_word2idx_limit_vocab
 
 # using ALS, what's the least # files to get correct analogies?
 # use this for word2vec training to make it faster
@@ -120,7 +120,7 @@ def fit(self, sentences, cc_matrix=None, learning_rate=1e-4, reg=0.1, xmax=100,
 
 
         costs = []
-        sentence_indexes = range(len(sentences))
+        #sentence_indexes = range(len(sentences))
         for epoch in range(epochs):
             delta = W.dot(U.T) + b.reshape(V, 1) + c.reshape(1, V) + mu - logX
             cost = ( fX * delta * delta ).sum()

nlp_class2/glove_svd.py

@@ -14,14 +14,13 @@
 from sklearn.decomposition import TruncatedSVD
 
 from datetime import datetime
-from sklearn.utils import shuffle
 from util import find_analogies
 
 
 import sys
 sys.path.append(os.path.abspath('..'))
 from rnn_class.util import get_wikipedia_data
-from rnn_class.brown import get_sentences_with_word2idx_limit_vocab, get_sentences_with_word2idx
+from rnn_class.brown import get_sentences_with_word2idx_limit_vocab
 
 
 class Glove:

nlp_class2/glove_tf.py

@@ -14,13 +14,13 @@
 import matplotlib.pyplot as plt
 
 from datetime import datetime
-from sklearn.utils import shuffle
+#from sklearn.utils import shuffle
 from util import find_analogies
 
 import sys
 sys.path.append(os.path.abspath('..'))
 from rnn_class.util import get_wikipedia_data
-from rnn_class.brown import get_sentences_with_word2idx_limit_vocab, get_sentences_with_word2idx
+from rnn_class.brown import get_sentences_with_word2idx_limit_vocab
 
 if tf.__version__.startswith('2'):
     tf.compat.v1.disable_eager_execution()
@@ -141,7 +141,7 @@ def fit(self, sentences, cc_matrix=None, learning_rate=1e-4, reg=0.1, xmax=100,
         session.run(init)
 
         costs = []
-        sentence_indexes = range(len(sentences))
+        #sentence_indexes = range(len(sentences))
         for epoch in range(epochs):
             c, _ = session.run((cost, train_op), feed_dict={tfLogX: logX, tffX: fX})
             print("epoch:", epoch, "cost:", c)
@@ -190,7 +190,7 @@ def main(we_file, w2i_file, use_brown=True, n_files=50):
 
     V = len(word2idx)
     model = Glove(100, V, 10)
-    model.fit(sentences, cc_matrix=cc_matrix, epochs=200)
+    model.fit(sentences, cc_matrix=cc_matrix, epochs=10000)
     model.save(we_file)
 
 

nlp_class2/logistic.py

@@ -47,7 +47,7 @@
 
 
   # train a logistic model
-  W = np.random.randn(V, V) / np.sqrt(V)
+  W = np.random.randn(V, V)/np.sqrt(V)
 
   losses = []
   epochs = 1
@@ -56,7 +56,7 @@
   def softmax(a):
     a = a - a.max()
     exp_a = np.exp(a)
-    return exp_a / exp_a.sum(axis=1, keepdims=True)
+    return exp_a/exp_a.sum(axis=1, keepdims=True)
 
   # what is the loss if we set W = log(bigram_probs)?
   W_bigram = np.log(bigram_probs)
@@ -85,19 +85,19 @@ def softmax(a):
       W = W - lr * inputs.T.dot(predictions - targets)
 
       # keep track of the loss
-      loss = -np.sum(targets * np.log(predictions)) / (n - 1)
+      loss = -np.sum(targets*np.log(predictions))/(n - 1)
       losses.append(loss)
 
       # keep track of the bigram loss
       # only do it for the first epoch to avoid redundancy
       if epoch == 0:
         bigram_predictions = softmax(inputs.dot(W_bigram))
-        bigram_loss = -np.sum(targets * np.log(bigram_predictions)) / (n - 1)
+        bigram_loss = -np.sum(targets*np.log(bigram_predictions))/(n - 1)
         bigram_losses.append(bigram_loss)
 
 
-      if j % 10 == 0:
-        print("epoch:", epoch, "sentence: %s/%s" % (j, len(sentences)), "loss:", loss)
+      if j%10 == 0:
+        print(f"epoch: {epoch}, sentence: {j}/{len(sentences)}, loss: {loss}")
       j += 1
 
   print("Elapsed time training:", datetime.now() - t0)
@@ -114,8 +114,8 @@ def smoothed_loss(x, decay=0.99):
     y = np.zeros(len(x))
     last = 0
     for t in range(len(x)):
-      z = decay * last + (1 - decay) * x[t]
-      y[t] = z / (1 - decay ** (t + 1))
+      z = decay*last + (1 - decay)*x[t]
+      y[t] = z/(1 - decay**(t + 1))
       last = z
     return y