From 4c156003e42bee35d049bd5f1a94c8fb6fcb8ace Mon Sep 17 00:00:00 2001
From: Payam Jome Yazdian <pjyazdian@users.noreply.github.com>
Date: Wed, 1 Feb 2023 10:41:50 -0800
Subject: [PATCH] Add files via upload

---
 scripts/Clustering.py                         | 1624 +++++++++++++++++
 scripts/creat_human-study.py                  |  126 ++
 .../data_preprocessor.cpython-37.pyc          |  Bin 0 -> 7591 bytes
 .../lmdb_data_loader.cpython-37.pyc           |  Bin 0 -> 19298 bytes
 scripts/data_loader/data_preprocessor.py      |  402 ++++
 scripts/data_loader/gesture_labels.txt        |  411 +++++
 scripts/data_loader/lmdb_data_loader.py       |  746 ++++++++
 scripts/inference.py                          |  189 ++
 scripts/inference_Autoencoder.py              |  564 ++++++
 scripts/inference_DAE.py                      |  659 +++++++
 scripts/inference_cluster2gesture.py          |  215 +++
 scripts/inference_text2embedding.py           |  774 ++++++++
 scripts/inference_text2embedding_GENEA.py     |  690 +++++++
 scripts/model/Autoencoder_VQVAE_model.py      | 1383 ++++++++++++++
 scripts/model/Autoencoder_model.py            |  388 ++++
 scripts/model/DAE_model.py                    |  470 +++++
 scripts/model/Helper_models.py                |  667 +++++++
 .../Autoencoder_VQVAE_model.cpython-37.pyc    |  Bin 0 -> 28281 bytes
 .../Autoencoder_model.cpython-37.pyc          |  Bin 0 -> 9284 bytes
 .../__pycache__/DAE_model.cpython-37.pyc      |  Bin 0 -> 8951 bytes
 .../__pycache__/Helper_models.cpython-37.pyc  |  Bin 0 -> 15447 bytes
 .../__pycache__/seq2seq_net.cpython-37.pyc    |  Bin 0 -> 7713 bytes
 .../seq2seq_with_cluster_model.cpython-37.pyc |  Bin 0 -> 2283 bytes
 .../text2embedding_GAN_model.cpython-37.pyc   |  Bin 0 -> 17664 bytes
 .../text2embedding_model.cpython-37.pyc       |  Bin 0 -> 14326 bytes
 .../model/__pycache__/vocab.cpython-37.pyc    |  Bin 0 -> 3856 bytes
 scripts/model/autoencoder_backup.py           |   38 +
 scripts/model/seq2seq_net.py                  |  257 +++
 scripts/model/seq2seq_with_cluster_model.py   |   71 +
 scripts/model/tcn.py                          |   64 +
 scripts/model/text2embedding_GAN_model.py     |  636 +++++++
 scripts/model/text2embedding_model.py         |  597 ++++++
 .../model/text2embedding_transformer_model.py |    0
 scripts/model/vocab.py                        |  130 ++
 scripts/pymo/__init__.py                      |    0
 .../pymo/__pycache__/__init__.cpython-37.pyc  |  Bin 0 -> 211 bytes
 scripts/pymo/__pycache__/data.cpython-37.pyc  |  Bin 0 -> 2748 bytes
 .../pymo/__pycache__/parsers.cpython-37.pyc   |  Bin 0 -> 7821 bytes
 .../__pycache__/preprocessing.cpython-37.pyc  |  Bin 0 -> 30154 bytes
 .../pymo/__pycache__/viz_tools.cpython-37.pyc |  Bin 0 -> 6993 bytes
 .../pymo/__pycache__/writers.cpython-37.pyc   |  Bin 0 -> 2634 bytes
 scripts/pymo/data.py                          |   53 +
 scripts/pymo/features.py                      |   43 +
 scripts/pymo/parsers.py                       |  260 +++
 scripts/pymo/preprocessing.py                 |  975 ++++++++++
 scripts/pymo/rotation_tools.py                |  220 +++
 scripts/pymo/viz_tools.py                     |  235 +++
 scripts/pymo/writers.py                       |   70 +
 scripts/save_clustered_fast.py                |  201 ++
 scripts/test_pymo.py                          |  138 ++
 .../test_transformers/transformers_model.py   | 1092 +++++++++++
 scripts/train.py                              |  189 ++
 scripts/train_Autoencoder.py                  |  568 ++++++
 scripts/train_DAE.py                          |  475 +++++
 scripts/train_autoencoder_VQVAE.py            |  505 +++++
 scripts/train_cluster2gesture.py              |  192 ++
 .../__pycache__/train_seq2seq.cpython-37.pyc  |  Bin 0 -> 8352 bytes
 scripts/train_eval/train_seq2seq.py           |  489 +++++
 scripts/train_gan.py                          |  240 +++
 scripts/train_stream_cluster.py               |  189 ++
 .../train_text2embedding(stream_gesture_).py  |  193 ++
 scripts/train_text2embedding.py               |  332 ++++
 scripts/trinity_data_to_lmdb.py               |  161 ++
 scripts/twh_dataset_to_lmdb.py                |  279 +++
 scripts/utils/Unityfier.py                    |   37 +
 .../__pycache__/average_meter.cpython-37.pyc  |  Bin 0 -> 1207 bytes
 .../__pycache__/data_utils.cpython-37.pyc     |  Bin 0 -> 2073 bytes
 .../__pycache__/data_utils_twh.cpython-37.pyc |  Bin 0 -> 2227 bytes
 .../__pycache__/train_utils.cpython-37.pyc    |  Bin 0 -> 2772 bytes
 .../__pycache__/vocab_utils.cpython-37.pyc    |  Bin 0 -> 1580 bytes
 scripts/utils/average_meter.py                |   23 +
 scripts/utils/data_utils.py                   |   48 +
 scripts/utils/data_utils_twh.py               |   65 +
 scripts/utils/train_utils.py                  |   85 +
 scripts/utils/vocab_utils.py                  |   57 +
 75 files changed, 18515 insertions(+)
 create mode 100644 scripts/Clustering.py
 create mode 100644 scripts/creat_human-study.py
 create mode 100644 scripts/data_loader/__pycache__/data_preprocessor.cpython-37.pyc
 create mode 100644 scripts/data_loader/__pycache__/lmdb_data_loader.cpython-37.pyc
 create mode 100644 scripts/data_loader/data_preprocessor.py
 create mode 100644 scripts/data_loader/gesture_labels.txt
 create mode 100644 scripts/data_loader/lmdb_data_loader.py
 create mode 100644 scripts/inference.py
 create mode 100644 scripts/inference_Autoencoder.py
 create mode 100644 scripts/inference_DAE.py
 create mode 100644 scripts/inference_cluster2gesture.py
 create mode 100644 scripts/inference_text2embedding.py
 create mode 100644 scripts/inference_text2embedding_GENEA.py
 create mode 100644 scripts/model/Autoencoder_VQVAE_model.py
 create mode 100644 scripts/model/Autoencoder_model.py
 create mode 100644 scripts/model/DAE_model.py
 create mode 100644 scripts/model/Helper_models.py
 create mode 100644 scripts/model/__pycache__/Autoencoder_VQVAE_model.cpython-37.pyc
 create mode 100644 scripts/model/__pycache__/Autoencoder_model.cpython-37.pyc
 create mode 100644 scripts/model/__pycache__/DAE_model.cpython-37.pyc
 create mode 100644 scripts/model/__pycache__/Helper_models.cpython-37.pyc
 create mode 100644 scripts/model/__pycache__/seq2seq_net.cpython-37.pyc
 create mode 100644 scripts/model/__pycache__/seq2seq_with_cluster_model.cpython-37.pyc
 create mode 100644 scripts/model/__pycache__/text2embedding_GAN_model.cpython-37.pyc
 create mode 100644 scripts/model/__pycache__/text2embedding_model.cpython-37.pyc
 create mode 100644 scripts/model/__pycache__/vocab.cpython-37.pyc
 create mode 100644 scripts/model/autoencoder_backup.py
 create mode 100644 scripts/model/seq2seq_net.py
 create mode 100644 scripts/model/seq2seq_with_cluster_model.py
 create mode 100644 scripts/model/tcn.py
 create mode 100644 scripts/model/text2embedding_GAN_model.py
 create mode 100644 scripts/model/text2embedding_model.py
 create mode 100644 scripts/model/text2embedding_transformer_model.py
 create mode 100644 scripts/model/vocab.py
 create mode 100644 scripts/pymo/__init__.py
 create mode 100644 scripts/pymo/__pycache__/__init__.cpython-37.pyc
 create mode 100644 scripts/pymo/__pycache__/data.cpython-37.pyc
 create mode 100644 scripts/pymo/__pycache__/parsers.cpython-37.pyc
 create mode 100644 scripts/pymo/__pycache__/preprocessing.cpython-37.pyc
 create mode 100644 scripts/pymo/__pycache__/viz_tools.cpython-37.pyc
 create mode 100644 scripts/pymo/__pycache__/writers.cpython-37.pyc
 create mode 100644 scripts/pymo/data.py
 create mode 100644 scripts/pymo/features.py
 create mode 100644 scripts/pymo/parsers.py
 create mode 100644 scripts/pymo/preprocessing.py
 create mode 100644 scripts/pymo/rotation_tools.py
 create mode 100644 scripts/pymo/viz_tools.py
 create mode 100644 scripts/pymo/writers.py
 create mode 100644 scripts/save_clustered_fast.py
 create mode 100644 scripts/test_pymo.py
 create mode 100644 scripts/test_transformers/transformers_model.py
 create mode 100644 scripts/train.py
 create mode 100644 scripts/train_Autoencoder.py
 create mode 100644 scripts/train_DAE.py
 create mode 100644 scripts/train_autoencoder_VQVAE.py
 create mode 100644 scripts/train_cluster2gesture.py
 create mode 100644 scripts/train_eval/__pycache__/train_seq2seq.cpython-37.pyc
 create mode 100644 scripts/train_eval/train_seq2seq.py
 create mode 100644 scripts/train_gan.py
 create mode 100644 scripts/train_stream_cluster.py
 create mode 100644 scripts/train_text2embedding(stream_gesture_).py
 create mode 100644 scripts/train_text2embedding.py
 create mode 100644 scripts/trinity_data_to_lmdb.py
 create mode 100644 scripts/twh_dataset_to_lmdb.py
 create mode 100644 scripts/utils/Unityfier.py
 create mode 100644 scripts/utils/__pycache__/average_meter.cpython-37.pyc
 create mode 100644 scripts/utils/__pycache__/data_utils.cpython-37.pyc
 create mode 100644 scripts/utils/__pycache__/data_utils_twh.cpython-37.pyc
 create mode 100644 scripts/utils/__pycache__/train_utils.cpython-37.pyc
 create mode 100644 scripts/utils/__pycache__/vocab_utils.cpython-37.pyc
 create mode 100644 scripts/utils/average_meter.py
 create mode 100644 scripts/utils/data_utils.py
 create mode 100644 scripts/utils/data_utils_twh.py
 create mode 100644 scripts/utils/train_utils.py
 create mode 100644 scripts/utils/vocab_utils.py

diff --git a/scripts/Clustering.py b/scripts/Clustering.py
new file mode 100644
index 0000000..bb3a390
--- /dev/null
+++ b/scripts/Clustering.py
@@ -0,0 +1,1624 @@
+import argparse
+import glob
+import math
+import pickle
+import pprint
+from pathlib import Path
+from collections import Counter
+
+import numpy as np
+import time
+import matplotlib.pyplot as plt
+from numpy import string_
+from sklearn import metrics
+from sklearn.datasets import make_blobs
+from sklearn.cluster import DBSCAN, KMeans, AgglomerativeClustering
+
+import torch
+from scipy.signal import savgol_filter
+import joblib as jl
+
+import utils
+from pymo.preprocessing import *
+from pymo.viz_tools import *
+from pymo.writers import *
+from sklearn.preprocessing import StandardScaler
+from tqdm import tqdm
+from utils.data_utils import SubtitleWrapper, normalize_string
+from utils.train_utils import set_logger
+
+from data_loader.data_preprocessor import DataPreprocessor
+
+from trinity_data_to_lmdb import process_bvh as process_bvh_trinity
+from twh_dataset_to_lmdb import process_bvh_rot_only_Taras as process_bvh_rot_only_Taras
+
+from twh_dataset_to_lmdb import process_bvh_test1 as process_bvh_rot_test1
+from inference_DAE import make_bvh_Trinity
+
+
+from inference_DAE import feat2bvh
+import os.path
+from sklearn.preprocessing import StandardScaler
+
+import matplotlib.pyplot as plt
+# from kneed import KneeLocator
+from sklearn.datasets import make_blobs
+from sklearn.cluster import KMeans
+from sklearn.metrics import silhouette_score
+from sklearn.preprocessing import StandardScaler
+from tslearn.clustering import TimeSeriesKMeans
+from mpl_toolkits.mplot3d import Axes3D
+from scipy.spatial.distance import pdist
+import random
+from scipy import stats
+
+
+from sklearn.decomposition import PCA
+from openTSNE import TSNE
+import seaborn as sns
+from scipy.ndimage.filters import gaussian_filter
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+DATA_TYPE = 'TWH'
+
+def generate_gestures_latent_dataset(args, DAE, rnn, bvh_file):
+
+    if DATA_TYPE=='Trinity':
+        poses, poses_mirror = process_bvh(bvh_file)
+    elif DATA_TYPE == 'TWH':
+        poses = process_bvh_rot_test1(bvh_file)
+        poses_mirror = poses
+    mean = np.array(args.data_mean).squeeze()
+    std = np.array(args.data_std).squeeze()
+    std = np.clip(std, a_min=0.01, a_max=None)
+    out_poses = (np.copy(poses_mirror) - mean) / std
+
+    target = torch.from_numpy(out_poses)
+    # target = torch.unsqueeze(target,2)
+    target = target.to(device).float()
+    reconstructed = []
+    # for i in range(len(out_poses)):
+    #     input = torch.unsqueeze(target[i],0)
+    #     current_out = pose_decoder(input)
+    #     reconstructed.append(current_out)
+    if DAE.encoder == None:
+        encoded = target
+    else:
+        encoded = DAE.encoder(target)
+    # encoded = torch.squeeze(encoded, 2)
+    # encoded = encoded.to('cpu')
+    # encoded = encoded.detach().numpy()
+    all_frames_from_rnn = None
+    all_sequences_poses_latent = []
+    for i in range(0, len(encoded), args.subdivision_stride):
+        current_dict = dict()
+        input_seq = encoded[i:i+args.n_poses]
+        if len(input_seq)<args.n_poses:
+            continue
+        # Todo: I did sanity check and 'original' should be normalized
+        current_dict['original'] = poses_mirror[i: i+args.n_poses]
+        current_dict['latent_linear'] = encoded[i: i + args.n_poses].detach().clone().to('cpu').numpy()
+
+
+
+
+        input_pre_seq = encoded[i]
+        output_seq = encoded[i:i+args.n_poses]
+
+        input_seq = torch.unsqueeze(input_seq, 0)
+        input_seq = input_seq.transpose(0, 1)
+        # Todo: use args
+        # use_drivitive=True
+        if args.use_derivitive == 'True':
+            diff = [(input_seq[n, :] - input_seq[n - 1, :]) for n in range(1, input_seq.shape[0])]
+            diff.insert(0, torch.zeros_like(input_seq[0, :]))
+            input_seq = torch.cat((input_seq, torch.stack(diff)), dim=2)
+
+
+        output_seq = torch.unsqueeze(output_seq, 0)
+        output_seq = output_seq.transpose(0, 1)
+        # reconstructed_rnn = torch.zeros(args.n_poses, output_seq.size(1), rnn.decoder.output_size)\
+        #     .to(output_seq.device)
+        reconstructed_rnn = torch.zeros(args.n_poses, output_seq.size(1), 135) \
+            .to(output_seq.device)
+        # run words through encoder
+        encoder_outputs, encoder_hidden = rnn.encoder(input_seq, None)
+
+        if rnn.VAE:
+            decoder_hidden = encoder_hidden[:rnn.decoder.n_layers]  # use last hidden state from encoder
+            # [2, 128, 200]
+            # print("decoder_hidden!!! org", decoder_hidden.shape)
+            decoder_hidden = decoder_hidden.transpose(1, 0).contiguous()  # [128, 2, 200]
+            decoder_hidden = torch.reshape(decoder_hidden, (decoder_hidden.shape[0], -1))
+            mean = rnn.VAE_fc_mean(decoder_hidden)
+            logvar = rnn.VAE_fc_std(decoder_hidden)
+            z = rnn.reparameterize(mean, logvar, train=False)
+            z = rnn.VAE_fc_decoder(z)
+            decoder_hidden = z.reshape(decoder_hidden.shape[0],
+                                       rnn.decoder.n_layers, -1)
+            decoder_hidden = decoder_hidden.transpose(1, 0).contiguous()
+        else:
+            decoder_hidden = encoder_hidden[:rnn.decoder.n_layers]  # use last hidden state from encoder
+        try:
+            if rnn.vq:
+                loss_vq, quantized, perplexity_vq, encodings = rnn.vq_layer(decoder_hidden)
+                encodings = encodings.detach().cpu().numpy()
+                quantized_indices = np.argmax(encodings, axis=1)
+                current_dict['quantized_indices'] = quantized_indices
+
+
+        except:
+            # print("Skip VQ: Trained using the old autoencoder without VQ option!")
+            pass
+        current_dict['latent_rnn'] = torch.squeeze(decoder_hidden.detach().clone(), 1).to('cpu').numpy()
+
+        all_sequences_poses_latent.append(current_dict)
+        # run through decoder one time step at a time
+    #     decoder_input = output_seq[0]  # initial pose from the dataset
+    #     reconstructed_rnn[0] = decoder_input
+    #
+    #     for t in range(1, rnn.n_frames):
+    #         decoder_output, decoder_hidden, _ = rnn.decoder(None, decoder_input, decoder_hidden, encoder_outputs,
+    #                                                          None)
+    #         reconstructed_rnn[t] = decoder_output
+    #
+    #         if t < rnn.n_pre_poses:
+    #             decoder_input = output_seq[t]  # next input is current target
+    #         else:
+    #             decoder_input = decoder_output  # next input is current prediction
+    #     if all_frames_from_rnn == None:
+    #         all_frames_from_rnn = reconstructed_rnn.transpose(0,1)
+    #     else:
+    #         all_frames_from_rnn = torch.cat((all_frames_from_rnn, reconstructed_rnn.transpose(0, 1)), 1)
+    #
+    #
+    # #     Todo: decode DAE
+    # all_frames_from_rnn = torch.squeeze(all_frames_from_rnn, 0)
+    # reconstructed_seq_DAE = DAE.decoder(all_frames_from_rnn)
+    # # reconstructed_seq_DAE = torch.squeeze(reconstructed_seq_DAE, 2)
+    # reconstructed_seq_DAE = reconstructed_seq_DAE.to('cpu')
+    # reconstructed_seq_DAE = reconstructed_seq_DAE.detach().numpy()
+
+
+
+    return all_sequences_poses_latent
+
+    '''
+        # smoothing motion transition
+        if len(out_list) > 0:
+            last_poses = out_list[-1][-args.n_pre_poses:]
+            out_list[-1] = out_list[-1][:-args.n_pre_poses]  # delete the last part
+
+            for j in range(len(last_poses)):
+                n = len(last_poses)
+                prev = last_poses[j]
+                next = out_seq[j]
+                out_seq[j] = prev * (n - j) / (n + 1) + next * (j + 1) / (n + 1)
+
+        out_list.append(out_seq)
+
+    print('Avg. inference time: {:.2} s'.format((time.time() - start) / num_subdivision))
+
+    # aggregate results
+    out_poses = np.vstack(out_list)
+    '''
+    return out_poses
+
+
+
+
+def make_VQ_Centers(checkpoint_path_DAE, checkpoint_path_rnn, save_path):
+    args, DAE, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+        checkpoint_path_DAE, device, 'DAE')
+    args, rnn, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+        checkpoint_path_rnn, device, 'autoencoder_vq')
+
+    for index, quantized in enumerate(rnn.vq_layer._embedding.weight):
+
+        # quantized = torch.reshape(quantized, [2, 1, 200]).contiguous()
+        quantized = torch.reshape(quantized, [args.n_layers, 1, args.hidden_size]).contiguous()
+        decoder_hidden = quantized.clone()
+
+        # run through decoder one time step at a time
+        decoder_input = torch.zeros(1, rnn.decoder.output_size).to(decoder_hidden.device)# initial pose from the dataset
+        reconstructed_rnn = torch.zeros(args.n_poses, 1, rnn.decoder.output_size).to(decoder_hidden.device)
+
+        reconstructed_rnn[0] = decoder_input
+
+        # repeat = 0
+        for rep in range(5):
+            decoder_output, decoder_hidden, _ = rnn.decoder(None, decoder_input, decoder_hidden, quantized,
+                                                            None)
+        all_frames_from_rnn = None
+        for t in range(1, rnn.n_frames):
+            if not rnn.autoencoder_conditioned:
+                decoder_input = torch.zeros_like(decoder_input)
+            decoder_output, decoder_hidden, _ = rnn.decoder(None, decoder_input, decoder_hidden, quantized,
+                                                            None)
+            reconstructed_rnn[t] = decoder_output
+
+            if t < rnn.n_pre_poses:
+                decoder_input = output_seq[t]  # next input is current target
+            else:
+                decoder_input = decoder_output  # next input is current prediction
+            if not rnn.autoencoder_conditioned:
+                decoder_input = torch.zeros_like(decoder_input)
+
+        if all_frames_from_rnn == None:
+            all_frames_from_rnn = reconstructed_rnn.transpose(0, 1)
+        else:
+            all_frames_from_rnn = torch.cat((all_frames_from_rnn, reconstructed_rnn.transpose(0, 1)), 1)
+
+        #     Todo: decode DAE
+
+        use_derivitive = False
+        all_frames_from_rnn = torch.squeeze(all_frames_from_rnn, 0)
+        if args.use_derivitive == True:
+            all_frames_from_rnn = all_frames_from_rnn[:, 0: all_frames_from_rnn.shape[1] // 2]
+        reconstructed_seq_DAE = DAE.decoder(all_frames_from_rnn)
+        # reconstructed_seq_DAE = torch.squeeze(reconstructed_seq_DAE, 2)
+        reconstructed_seq_DAE = reconstructed_seq_DAE.to('cpu')
+        reconstructed_seq_DAE = reconstructed_seq_DAE.detach().numpy()
+
+        # Todo: save the reconstructed
+        # unnormalize
+        mean = np.array(args.data_mean).squeeze()
+        std = np.array(args.data_std).squeeze()
+        std = np.clip(std, a_min=0.01, a_max=None)
+        reconstructed_seq_DAE = np.multiply(reconstructed_seq_DAE, std) + mean
+
+
+        os.makedirs(save_path + '/Centers/', exist_ok=True)
+        filename_prefix = 'Center_{}'.format(index)
+        if DATA_TYPE=='Trinity':
+            make_bvh(save_path + '/Centers/', filename_prefix, reconstructed_seq_DAE)
+        elif DATA_TYPE=='TWH':
+            make_bvh_Trinity(save_path + '/Centers/', filename_prefix, reconstructed_seq_DAE)
+
+def maake_dataset(checkpoint_path_DAE, checkpoint_path_rnn, pickle_adress):
+    args, DAE, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+        checkpoint_path_DAE, device, 'DAE')
+    args, rnn, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+        checkpoint_path_rnn, device, 'autoencoder_vq')
+    pprint.pprint(vars(args))
+    save_path = '../output/infer_sample'
+    os.makedirs(save_path, exist_ok=True)
+
+    # load lang_model
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    with open(vocab_cache_path, 'rb') as f:
+        lang_model = pickle.load(f)
+
+    # prepare input
+    # transcript = SubtitleWrapper(transcript_path).get()
+
+    # inference
+    all_seq_real_latent = []
+    gesture_path = '/local-scratch/pjomeyaz/rosie_gesture_benchmark/cloned/Clustering/must/GENEA/Co-Speech_Gesture_Generation/dataset/dataset_v1/trn/bvh'
+
+    bvh_files = sorted(glob.glob(gesture_path + "/*.bvh"))
+
+    count = 0
+
+    for bvh_file in tqdm(bvh_files):
+        count += 1
+        if count > 20:
+            print("*** More than 100 files")
+            break
+        name = os.path.split(bvh_file)[1][:-4]
+        print("Processing", name)
+
+        # bvh_file = '/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Motion/TestSeq001.bvh'
+
+        returned_org_latent = generate_gestures_latent_dataset(args, DAE, rnn, bvh_file)
+        all_seq_real_latent.append(returned_org_latent)
+        # break
+    pickle.dump(all_seq_real_latent, open(pickle_adress, 'wb'))
+
+    return
+    '''
+    # unnormalize
+    mean = np.array(args.data_mean).squeeze()
+    std = np.array(args.data_std).squeeze()
+    std = np.clip(std, a_min=0.01, a_max=None)
+    reconstructed = np.multiply(reconstructed, std) + mean
+    org_poses = np.multiply(org_poses, std) + mean
+
+    # ........ infer
+
+
+
+    # make a BVH
+    filename_prefix = '{}'.format("test_original")
+    make_bvh(save_path, filename_prefix, org_poses)
+    filename_prefix = '{}'.format("test_reconstructed")
+    make_bvh(save_path, filename_prefix, reconstructed)
+    '''
+
+def maake_dataset_for_inference(checkpoint_path_DAE, checkpoint_path_rnn, bvh_file):
+    args, DAE, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+        checkpoint_path_DAE, device, 'DAE')
+    args, rnn, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+        checkpoint_path_rnn, device, 'autoencoder_vq')
+    pprint.pprint(vars(args))
+    save_path = '../output/infer_sample'
+    os.makedirs(save_path, exist_ok=True)
+
+    # load lang_model
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    with open(vocab_cache_path, 'rb') as f:
+        lang_model = pickle.load(f)
+
+    # prepare input
+    # transcript = SubtitleWrapper(transcript_path).get()
+
+    # inference
+    all_seq_real_latent = []
+    gesture_path = '/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Training_data/Motion'
+
+    # bvh_files = sorted(glob.glob(gesture_path + "/*.bvh"))
+
+    name = os.path.split(bvh_file)[1][:-4]
+    print("Processing", name)
+
+    # bvh_file = '/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Motion/TestSeq001.bvh'
+
+    if args.autoencoder_vq == 'True':
+        returned_org_latent = generate_gestures_latent_dataset(args, DAE, rnn, bvh_file)
+    else:
+        returned_org_latent = generate_gestures_latent_dataset(args, DAE, rnn, bvh_file)
+    return returned_org_latent
+
+
+def make_bvh(save_path, filename_prefix, poses):
+    writer = BVHWriter()
+    pipeline = jl.load('../resource/data_pipe.sav')
+
+    # smoothing
+    n_poses = poses.shape[0]
+    out_poses = np.zeros((n_poses, poses.shape[1]))
+
+    try:
+        for i in range(poses.shape[1]):
+            out_poses[:, i] = savgol_filter(poses[:, i], 15, 2)  # NOTE: smoothing on rotation matrices is not optimal
+    except:
+        pass
+    # rotation matrix to euler angles
+    out_poses = out_poses.reshape((out_poses.shape[0], -1, 9))
+    out_poses = out_poses.reshape((out_poses.shape[0], out_poses.shape[1], 3, 3))
+    out_euler = np.zeros((out_poses.shape[0], out_poses.shape[1] * 3))
+    for i in range(out_poses.shape[0]):  # frames
+        r = R.from_matrix(out_poses[i])
+        out_euler[i] = r.as_euler('ZXY', degrees=True).flatten()
+
+    bvh_data = pipeline.inverse_transform([out_euler])
+
+    out_bvh_path = os.path.join(save_path, filename_prefix + '_generated.bvh')
+    with open(out_bvh_path, 'w') as f:
+        writer.write(bvh_data[0], f)
+
+Flag_VQ = True
+
+
+def calculate_distances(data_latent_rnn_ndarray):
+    print("Calculatiing neighbour feature distance...")
+    # print("Skipping neighbour feature distance calculation...")
+    # return
+    # calculate representation quality
+    # Implementation of representation space metric
+    str_rep = '=====Representation similarity at the embedding space.=====\n\n'
+    all_dist = pdist(data_latent_rnn_ndarray)
+    avg_dist_total = np.mean(all_dist)
+    for method in ['all', 'random']:
+        print("Latent calculation method: ", method)
+        if method == 'all':
+            indicies = range(2, len(data_latent_rnn_ndarray) - 2)
+        else:
+            indicies = random.sample(range(2, len(data_latent_rnn_ndarray) - 2), 1000)
+
+        neighbour_20fs = []
+        neighbour_10fs = []
+        # for i in range(2, len(data_latent_rnn_ndarray) - 2):
+        for i in tqdm(indicies):
+            n20s = (np.sqrt(np.sum((data_latent_rnn_ndarray[i - 1]
+                                  - data_latent_rnn_ndarray[i]) ** 2, axis=0)) \
+                  + np.sqrt(np.sum((data_latent_rnn_ndarray[i + 1]
+                                    - data_latent_rnn_ndarray[i]) ** 2, axis=0))
+                  ) / 2
+            neighbour_20fs.append(n20s)
+
+            n10s = (np.sqrt(np.sum((data_latent_rnn_ndarray[i - 2]
+                                  - data_latent_rnn_ndarray[i]) ** 2, axis=0)) \
+                  + np.sqrt(np.sum((data_latent_rnn_ndarray[i + 2]
+                                    - data_latent_rnn_ndarray[i]) ** 2, axis=0))
+                  ) / 2
+            neighbour_10fs.append(n10s)
+
+        avg_20fs = np.mean(neighbour_20fs)
+        std_20fs = np.std(neighbour_20fs)
+
+        avg_10fs = np.mean(neighbour_10fs)
+        std_10fs = np.std(neighbour_10fs)
+
+
+
+
+        neighbour_20fs /= avg_dist_total
+        neighbour_10fs /= avg_dist_total
+        normal_avg_20fs = np.mean(neighbour_20fs)
+        normal_std_20fs = np.std(neighbour_20fs)
+        normal_avg_10fs = np.mean(neighbour_10fs)
+        normal_std_10fs = np.std(neighbour_10fs)
+
+        str_rep += '\n\n==========================\nMethod: {}\n'.format(method)
+        str_rep += "Representation Similarity metric\n" + \
+                  "avg_20fs={} -- std_20fs={}\navg_10fs={} -- std_10fs={}\n" \
+                      .format(avg_20fs, std_20fs, avg_10fs, std_10fs)
+        str_rep += "\nStandardized:\n" + "average distance total: " \
+                   + str(avg_dist_total) \
+                   + "\nstandardized distances\n" \
+                   + "normal_avg_20fs={} -- normal_std_20fs={}\nnormal_avg_10fs={} -- normal_std_10fs={}\n" \
+                       .format(normal_avg_20fs, normal_std_20fs, normal_avg_10fs, normal_std_10fs)
+        str_rep += '\n\n'
+    print(str_rep)
+    metric_file = open(save_result_path + "/Rep_distance.txt", 'w')
+    metric_file.write(str_rep)
+    metric_file.flush()
+    metric_file.close()
+
+
+
+def cluster(loaded, eps, min_samples, save_result_path):
+
+    data_latent_rnn = []
+    data_latent_linear = np.zeros([len(loaded), loaded[0]['latent_linear'].shape[0],
+                                   loaded[0]['latent_linear'].shape[1]])
+    data_latent_linear_listwise = []
+    data_original = []
+    data_quantized_indicies = []
+    count = len(loaded)
+    # count = 4000
+    for i in range(count):
+
+
+        # 1
+
+        current_latent_linear = loaded[i]['latent_linear']
+        current_latent_rnn = loaded[i]['latent_rnn']
+        current_original = loaded[i]['original']
+        if Flag_VQ:
+            current_quantized_indicies = loaded[i]['quantized_indices'][0]
+
+        if len(current_original) !=len(loaded[0]['original']):
+            continue
+
+
+        # 2
+        # current_latent_linear = np.hstack(current_latent_linear)
+        current_latent_rnn = np.hstack(current_latent_rnn)
+        # current_original = np.hstack(current_original)
+
+        # 3
+        data_latent_linear[i] = (current_latent_linear)
+        data_latent_linear_listwise.append(current_latent_linear)
+        data_latent_rnn.append(current_latent_rnn)
+        data_original.append(current_original)
+        if Flag_VQ:
+            data_quantized_indicies.append(current_quantized_indicies)
+    # Should be constructed here since it is not obvious how many we will have at the end.
+    data_latent_linear = np.zeros([len(data_latent_linear_listwise), loaded[0]['latent_linear'].shape[0],
+                                   loaded[0]['latent_linear'].shape[1]])
+    for i in range(len(data_latent_linear_listwise)):
+        data_latent_linear[i] = data_latent_linear_listwise[i]
+
+    data_latent_rnn_ndarray = np.array(data_latent_rnn)
+
+    # Calculate the distance at the representation embedding space
+    # calculate_distances(data_latent_rnn_ndarray)
+
+
+    # scaler = StandardScaler()
+    # scaled_features = scalar.fit_transform(data)
+
+    # 1. clustering parameters.
+    kmeans_kwargs = {
+        "init": "random",
+        "n_init": 10,
+        "max_iter": 300,
+        "random_state": 42,
+        }
+
+    if False:
+        # 2. Analysis 1:
+        sse = []
+        for k in range(1, 11):
+            kmeans = KMeans(n_clusters=k, **kmeans_kwargs)
+            kmeans.fit(data_latent_rnn)
+            sse.append(kmeans.inertia_)
+
+        plt.style.use("fivethirtyeight")
+        plt.plot(range(1, 11), sse)
+        plt.xticks(range(1, 11))
+        plt.xlabel("Number of Clusters")
+        plt.ylabel("SSE")
+        plt.show()
+
+    # 3. Analysis 2
+    # A list holds the silhouette coefficients for each k
+    silhouette_coefficients = []
+    # Notice you start at 2 clusters for silhouette coefficient
+    if False:
+        for k in range(50, 400, 25):
+            kmeans = KMeans(n_clusters=k, **kmeans_kwargs)
+            kmeans.fit(data_latent_rnn)
+            score = silhouette_score(data_latent_rnn, kmeans.labels_)
+            silhouette_coefficients.append(score)
+        plt.figure()
+        plt.style.use("fivethirtyeight")
+        plt.plot(range(50, 400, 25), silhouette_coefficients)
+        plt.xticks(range(50, 400, 25))
+        plt.xlabel("Number of Clusters")
+        plt.ylabel("Silhouette Coefficient")
+        plt.show()
+
+    k_component = 40
+
+    if False:
+        # 4. Dynamic Time Warping.
+        # X = random_walks(n_ts=50, sz=32, d=1)
+        # 4.1 Euclidean distance
+        print("4.1 Euclidean distance")
+
+        # print(data_latent_linear[0].shape)
+
+        km = TimeSeriesKMeans(n_clusters=k_component, metric="euclidean", max_iter=5,
+        random_state = 0).fit(data_latent_linear)
+        km.cluster_centers_.shape
+        labels = km.labels_
+
+
+        # 4.1.1 save results
+        print("4.1.1 save results")
+        for i in tqdm(range(0, len(data_latent_linear), 10)):
+            directory = str(save_result_path) + '/4.1/' + str(labels[i])
+            if not os.path.exists(directory):
+                os.makedirs(directory)
+            make_bvh(directory, str(i), data_original[i])
+
+    if False:
+        # 4.2 DTW distance
+        print(".2 DTW distance")
+        km_dba = TimeSeriesKMeans(n_clusters=k_component, metric="dtw", max_iter=5,
+                                  max_iter_barycenter=5,
+                                  random_state=0).fit(data_latent_linear)
+        km_dba.cluster_centers_.shape
+        labels = km_dba.labels_
+
+        # 4.2.1 save results
+        print("4.2.1 save results")
+        for i in tqdm(range(0, len(data_latent_linear), 10)):
+            directory = str(save_result_path) + '/4.2/' + str(labels[i])
+            if not os.path.exists(directory):
+                os.makedirs(directory)
+            make_bvh(directory, str(i), data_original[i])
+
+    if False:
+    # 4.3 softDTW distance
+        print("4.3 softDTW distance")
+        km_sdtw = TimeSeriesKMeans(n_clusters=k_component, metric="softdtw", max_iter=5,
+                                    max_iter_barycenter = 5,
+                                    metric_params = {"gamma": .5},
+                                    random_state = 0).fit(data_latent_linear)
+        km_sdtw.cluster_centers_.shape
+        labels = km_sdtw.labels_
+
+        # 4.3.1 save results
+        print("4.3.1 save results")
+        for i in tqdm(range(0, len(data_latent_linear), 10)):
+            directory = str(save_result_path) + '/4.3/' + str(labels[i])
+            if not os.path.exists(directory):
+                os.makedirs(directory)
+            make_bvh(directory, str(i), data_original[i])
+
+    # 4.4 RNN based clustering
+    print("4.4 RNN based")
+
+    scaler = StandardScaler()
+    data_latent_rnn_ndarray = data_latent_rnn_ndarray.squeeze()
+    scaler.fit(data_latent_rnn_ndarray)
+    # Todo: check which one is better? Unnormalized showed better results.
+    # normalized_data = scaler.transform(data_latent_rnn_ndarray)
+    normalized_data = data_latent_rnn_ndarray
+
+
+    # 4.4.0 Algorithm
+    Algorithm = 'kmeans'
+    k_component = 300
+    if Algorithm == 'kmeans':
+
+        # Save address:
+        km_address = (save_result_path) + '/'
+        os.makedirs(km_address, exist_ok=True)
+        km_address += 'kmeans_model.pk'
+
+        if os.path.exists(km_address):
+            km_rnn = pickle.load(open(km_address, 'rb'))
+            print("Kmeans Loaded!")
+
+        else: # train kmeans
+            km_rnn = KMeans(n_clusters=k_component,  max_iter=2500,
+                                       random_state=0).fit(normalized_data)
+            print("Kmeans trained!")
+            km_rnn.cluster_centers_.shape
+            # Save model
+            # km_rnn.to_pickle('../output/clustering_results/kmeans_model.pk')
+            # pickle.dump(km_rnn, open('../output/clustering_results/kmeans_model.pk', 'wb'))
+            pickle.dump(km_rnn, open(km_address, 'wb'))
+        labels = km_rnn.labels_
+        labels = list(labels)
+
+
+    elif Algorithm == 'mdp':
+        X = normalized_data.T
+        N = X.shape[1]
+        # Set up Normal-Wishart MAP-DP prior parameters
+        N0 = 0.5  # Prior count (concentration parameter)
+        m0 = X.mean(1)[:, None]  # Normal-Wishart prior mean
+        a0 = 10  # Normal-Wishart prior scale
+        c0 = 10 / float(N)  # Normal-Wishart prior degrees of freedom
+        B0 = np.diag(1 / (0.05 * X.var(1)))  # Normal-Wishart prior precision
+        # # Run MAPDP to convergence
+
+        mu, labels, k_component, E = mapdp_nw(X, N0, m0, a0, c0, B0)
+    #
+    elif Algorithm == 'dbscan':
+        db = DBSCAN(eps=0.05, min_samples=10).fit(normalized_data)
+        core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
+        core_samples_mask[db.core_sample_indices_] = True
+        labels = db.labels_
+
+        # Number of clusters in labels, ignoring noise if present.
+        k_component = len(set(labels)) - (1 if -1 in labels else 0)
+        n_noise_ = list(labels).count(-1)
+    elif Algorithm == 'AgglomerativeClustering':
+        clustering  = AgglomerativeClustering(linkage='ward',
+                                              n_clusters=k_component).fit(normalized_data)
+        labels = clustering.labels_
+    # 4.4.1 save results
+
+
+
+    # Plot statistic
+    objects = [str(x) for x in range(0, k_component)]
+    y_pos = np.arange(len(objects))
+    # performance = [labels.count(i) for i in labels]
+
+    a = dict(Counter(labels))
+
+    # Sanity check
+    check4 = 0
+    # for uuu in range(len(labels)):
+    #     if labels[uuu] == 4:
+    #         check4 = check4 +1
+    # Passed
+
+    plot_Portion = False
+    if plot_Portion:
+        total = 0
+        # for saving the portion
+        for index in a:
+            total += a[index]
+        portion = []
+        for i in range(0, k_component):
+            if i == -1:
+                continue
+            portion.append(a[i]/total)
+        portion = np.array(portion)
+        performance = []
+        for i in range(0, k_component):
+            performance.append(a[i])
+        # plt.figure(figsize=(100,400))
+        plt.bar(y_pos, performance, align='center', alpha=0.5, width=0.8)
+        plt.xticks(y_pos, objects, size='small')
+        plt.xticks(rotation=90)
+        plt.ylabel('occurrence')
+        plt.title('clustered samples')
+
+        plt.show()
+
+
+    print("4.4.1 save results")
+
+    save = True
+    # save = True
+
+    # Save clusters' centers
+    if False:
+        if Algorithm =='kmeans':
+            for c_i, clusters_centers in enumerate(km_rnn.cluster_centers_):
+                directory = str(save_result_path) + '/4.4/' + str(c_i)
+                if not os.path.exists(directory):
+                    os.makedirs(directory)
+
+                min_dist_1 = np.sqrt(np.mean(
+                    km_rnn.transform(
+                        data_latent_rnn_ndarray[0, :].reshape(1, -1)
+                    )[:, c_i]**2))
+                min_index_1 = 0
+
+                min_dist_2 = np.sqrt(np.mean(
+                    km_rnn.transform(
+                        data_latent_rnn_ndarray[1, :].reshape(1, -1)
+                    )[:, c_i] ** 2))
+                min_index_2 = 1
+
+                if min_dist_1 > min_dist_2:
+                    min_dist_1, min_dist_2 = min_dist_2, min_dist_1
+                    min_index_1, min_index_2 = min_index_2, min_index_1
+
+                for s_i in tqdm(range(0, len(data_latent_linear))):
+
+                    # check if the instance belongs to c_index cluster
+                    if labels[s_i] != c_i:
+                        continue
+
+                    # array.reshape(1, -1) if it contains a single sample
+                    reshaped = data_latent_rnn_ndarray[s_i, :]
+                    reshaped = reshaped.reshape(1, -1)
+                    current_distance = km_rnn.transform(reshaped)[:, c_i]
+                    current_distance = np.mean(current_distance**2)
+                    current_distance = np.sqrt(current_distance)
+                    if current_distance < min_dist_1:
+                        min_index_2 = min_dist_1
+                        min_index_2 = min_index_1
+                        min_dist_1 = current_distance
+                        min_index_1 = s_i
+                    elif current_distance < min_dist_2:
+                        min_dist_2 = current_distance
+                        min_index_2 = s_i
+                if DATA_TYPE=='Trinity':
+                    make_bvh(directory, "Closest1-" + str(min_index_1), data_original[min_index_1])
+                    make_bvh(directory, "Closest2-" + str(min_index_2), data_original[min_index_2])
+                elif DATA_TYPE=='TWH':
+                    make_bvh_Trinity(directory, "Closest1-" + str(min_index_1), data_original[min_index_1])
+                    make_bvh_Trinity(directory, "Closest2-" + str(min_index_2), data_original[min_index_2])
+
+
+    if True:
+        if Flag_VQ:
+            for i in tqdm(range(0, 7000, 1)):
+                directory = str(save_result_path) + '/VQ_clusters/' + str(data_quantized_indicies[i])
+                if not os.path.exists(directory):
+                    os.makedirs(directory)
+                if DATA_TYPE=='Trinity':
+                    make_bvh(directory, str(i), data_original[i])
+                elif DATA_TYPE=='TWH':
+                    make_bvh_Trinity(directory, str(i), data_original[i])
+        # for i in tqdm(range(0, len(data_latent_linear), 1)):
+        for i in tqdm(range(0, 4000, 1)):
+            directory = str(save_result_path) + '/kmeans_clusters/' + str(labels[i])
+            if not os.path.exists(directory):
+                os.makedirs(directory)
+            if DATA_TYPE=='Trinity':
+                make_bvh(directory, str(i), data_original[i])
+            elif DATA_TYPE=='TWH':
+                make_bvh_Trinity(directory, str(i), data_original[i])
+
+
+    # 5. Here I will implent flowgmm based clustering
+    # todo: implement it.
+
+
+
+    # n. Visualization when?
+    # Prepare for clustrting visualization:
+    directory = str(save_result_path)
+    # if not os.path.exists(directory):
+    #     os.makedirs(directory)
+
+    portion = None
+    if True:
+        if Flag_VQ:
+            plot_tsne(args, normalized_data, scaler, k_component, np.array(data_quantized_indicies),
+                      directory + '/transforms.pkl', True, portion)
+        else:
+            plot_tsne(args, normalized_data, scaler, k_component, km_rnn.labels_,
+                      directory + '/transforms.pkl', True, portion)
+
+    # Histogram_clusters(args, km_rnn, normalized_data, scaler, k_component, km_rnn.labels_, '../output/transforms.pkl', False, portion)
+    # Metrics_analysis(args, km_rnn, normalized_data, scaler,
+    #                  k_component, km_rnn.labels_, '../output/transforms.pkl', False)
+
+
+    # 3D
+    exit()
+    X_embedded = TSNE(n_components=3).fit_transform(normalized_data)
+
+    sns.set(style="darkgrid")
+    fig = plt.figure()
+    ax = fig.add_subplot(111, projection='3d')
+    ax.set_xlabel("Dim1")
+    ax.set_ylabel("Dim2")
+    ax.set_zlabel("Dim3")
+    ax.scatter(X_embedded[:, 0], X_embedded[:, 1], X_embedded[:, 2], c=km_rnn.labels_)
+    plt.show()
+    exit()
+    pickle.dump(kmeans.labels_, open('../output/kmeans_labels.bin', 'wb'))
+
+    # # DECODE GESTURES BASED ON CLUSTERS
+    # for i in range(len(data)):
+    #     current_data = data[i]
+    #     current_data = torch.from_numpy(current_data).to(device)
+    #     decoder_hidden = current_data
+
+
+    exit()
+
+
+
+    # Compute DBSCAN
+
+    db = DBSCAN(eps=eps, min_samples=10).fit(scaled_features)
+    core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
+    core_samples_mask[db.core_sample_indices_] = True
+    labels = db.labels_
+
+    # Number of clusters in labels, ignoring noise if present.
+    k_component = len(set(labels)) - (1 if -1 in labels else 0)
+    n_noise_ = list(labels).count(-1)
+
+    print('Estimated number of clusters: %d' % n_clusters_)
+    print('Estimated number of noise points: %d' % n_noise_)
+    # print("Homogeneity: %0.3f" % metrics.homogeneity_score(labels_true, labels))
+    # print("Completeness: %0.3f" % metrics.completeness_score(labels_true, labels))
+    # print("V-measure: %0.3f" % metrics.v_measure_score(labels_true, labels))
+    # print("Adjusted Rand Index: %0.3f"
+    #       % metrics.adjusted_rand_score(labels_true, labels))
+    # print("Adjusted Mutual Information: %0.3f"
+    #       % metrics.adjusted_mutual_info_score(labels_true, labels))
+    # print("Silhouette Coefficient: %0.3f"
+    #       % metrics.silhouette_score(X, labels))
+
+    # #############################################################################
+    # Plot result
+
+
+    # Black removed and is used for noise instead.
+    unique_labels = set(labels)
+    colors = [plt.cm.Spectral(each)
+              for each in np.linspace(0, 1, len(unique_labels))]
+    for k, col in zip(unique_labels, colors):
+        if k == -1:
+            # Black used for noise.
+            col = [0, 0, 0, 1]
+
+        class_member_mask = (labels == k)
+
+        xy = data[class_member_mask & core_samples_mask]
+        plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
+                 markeredgecolor='k', markersize=14)
+
+        xy = X[class_member_mask & ~core_samples_mask]
+        plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col),
+                 markeredgecolor='k', markersize=6)
+
+    plt.title('Estimated number of clusters: %d' % n_clusters_)
+    plt.show()
+
+def plot_tsne(args, normalized_data, scalar, k_component, labels_, save_adress, rewrite, portion):
+
+
+    if rewrite:
+        # PCA
+        pca = PCA(n_components=50)
+        priciple_components = pca.fit(normalized_data)
+        normalized_data = priciple_components.transform(normalized_data)
+        # TSNE
+
+        MyTSNE = TSNE(
+            n_components=2,
+            perplexity=30,
+            metric='euclidean',
+            n_jobs=8,
+            verbose=True
+        )
+        X_embedded = MyTSNE.fit(normalized_data)
+
+        #     Save
+        stats = {'pca': pca, 'tsne': X_embedded, 'scalar': scalar, 'portion': portion}
+        pickle.dump(stats, open(save_adress, 'wb'))
+    else:
+        # PCA
+        loaded = pickle.load(open(save_adress, 'rb'))
+        pca, MyTSNE = loaded['pca'], loaded['tsne']
+        normalized_data = pca.transform(normalized_data)
+        # TSNE
+        X_embedded = MyTSNE.transform(normalized_data)
+
+
+    plt.figure(figsize=(16, 10))
+    palette = sns.color_palette("Set2", k_component)
+    # 2D
+    min = np.min(labels_)
+    max = np.max(labels_)
+    a_set = set(labels_)
+    number_of_unique_values = len(a_set)
+    sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1],
+                    hue=labels_, legend=False, palette=sns.color_palette('Set2', n_colors=number_of_unique_values))
+
+    address = os.path.dirname(args.ckpt_path_Autoencode) + '/TSNE.png'
+    plt.savefig(address)
+    plt.show()
+
+
+
+    # Hitmap
+    x_lim, y_lim = 60, 60
+
+    heatmap, x_edge, y_edge = np.histogram2d(X_embedded[:, 0], X_embedded[:, 1], bins=100)
+    extent = [x_edge[0], x_edge[-1], y_edge[0], y_edge[-1]]
+    extent = [-y_lim, y_lim, -x_lim, x_lim]
+    plt.clf()
+    sigma = 2
+    for sigma in range(2, 3):
+        plt.figure(figsize=(16, 10))
+        heatmap_filtered = gaussian_filter(heatmap, sigma)
+        plt.imshow(heatmap_filtered.T, extent=extent, origin='lower')
+        # plt.colorbar(plt.pcolor(heatmap))
+        # plt.pcolor(heatmap)
+        plt.colorbar()
+        plt.title('Heatmap of representation dataset, Sigma = ' + str(sigma))
+
+        address = os.path.dirname(args.ckpt_path_Autoencode) + '/Hitmap.png'
+        plt.savefig(address)
+
+        plt.show()
+        plt.clf()
+
+
+    return
+
+    # # Plot generated
+    bvh_file1 = '/local-scratch/pjomeyaz/rosie_gesture_benchmark/cloned/Clustering/must/Co-Speech_Gesture_Generation/output' \
+               '/Experiment/Objective/BVH_submitted/Ground_Truth/TestSeq001.bvh'
+    bvh_file2 = '/local-scratch/pjomeyaz/rosie_gesture_benchmark/cloned/Clustering/must/Co-Speech_Gesture_Generation/output' \
+               '/Experiment/Objective/BVH_submitted/Cond_BA/TestSeq001.bvh'
+    bvh_file3 = '/local-scratch/pjomeyaz/rosie_gesture_benchmark/cloned/Clustering/must/Co-Speech_Gesture_Generation/output' \
+               '/Experiment/Objective/BVH_submitted/Cond_BT/TestSeq001.bvh'
+    bvh_file4 = '/local-scratch/pjomeyaz/rosie_gesture_benchmark/cloned/Clustering/must/Co-Speech_Gesture_Generation/output' \
+               '/Experiment/Objective/BVH_submitted/Cond_SD/TestSeq001.bvh'
+    bvh_file5 = '/local-scratch/pjomeyaz/rosie_gesture_benchmark/cloned/Clustering/must/Co-Speech_Gesture_Generation/output' \
+               '/Experiment/Objective/BVH_submitted/proposed_not_smooth/TestSeq001.bvh'
+
+    bvh_files = [bvh_file1, bvh_file2, bvh_file3, bvh_file4, bvh_file5]
+    for bvh_file in bvh_files:
+        latent_org = maake_dataset_for_inference(args.ckpt_path_DAE, args.ckpt_path_Autoencode, bvh_file)
+        all_latents = []
+        for item in latent_org:
+            current_rnn = item['latent_rnn']
+            current_rnn = np.hstack(current_rnn)
+            all_latents.append(current_rnn)
+        all_latents = np.array(all_latents)
+        all_latents = scalar.transform(all_latents) #normalization
+        all_latents = pca.transform(all_latents)
+        all_latents = MyTSNE.transform(all_latents)
+        plt.scatter(all_latents[:, 0], all_latents[:, 1], c='r')
+        plt.xlim(-y_lim, y_lim)
+        plt.ylim(-x_lim, x_lim)
+        plt.title(bvh_file[-30:-15])
+        plt.show()
+        # Heatmap of it
+        heatmap, x_edge, y_edge = np.histogram2d(all_latents[:, 0], all_latents[:, 1], bins=100)
+        extent = [x_edge[0], x_edge[-1], y_edge[0], y_edge[-1]]
+        extent = [-y_lim, y_lim, -x_lim, x_lim]
+        plt.clf()
+        sigma = 1
+        heatmap_filtered = gaussian_filter(heatmap, sigma)
+        plt.imshow(heatmap_filtered.T, extent=extent, origin='lower')
+        plt.colorbar()
+        plt.title('Heatmap of generated infer, Sigma = ' + str(sigma))
+        plt.xlabel(bvh_file[-20:])
+        plt.show()
+        plt.clf()
+
+
+
+def Histogram_clusters(args, km, normalized_data, scalar, k_component, labels_, save_adress, rewrite, portion):
+
+    from sklearn.decomposition import PCA
+    from openTSNE import TSNE
+    import seaborn as sns
+    from scipy.ndimage.filters import gaussian_filter
+
+
+    # # Plot generated
+    dir_submitted = '/local-scratch/pjomeyaz/rosie_gesture_benchmark/cloned/Clustering/must/Co-Speech_Gesture_Generation/output' \
+               '/Experiment/Objective/BVH_submitted/'
+    conditions = ['Ground_Truth', 'Cond_BA', 'Cond_BT', 'Cond_SD',
+                  'proposed_not_smooth', 'proposed_smooth_long',
+                  'proposed_14']
+    Hist_GT = None
+    str_out = ''
+
+    for cond_dir in conditions:
+        bvh_files = sorted(glob.glob(os.path.join(dir_submitted + cond_dir, '*.bvh')))
+        all_latents = []
+        for bvh_file in bvh_files:
+            latent_org = maake_dataset_for_inference(args.ckpt_path_DAE, args.ckpt_path_Autoencode, bvh_file)
+            for item in latent_org:
+                current_rnn = item['latent_rnn']
+                current_rnn = np.hstack(current_rnn)
+                all_latents.append(current_rnn)
+
+        all_latents = np.array(all_latents)
+        all_latents = scalar.transform(all_latents) #normalization
+
+        labels = km.predict(all_latents)
+        labels_dict = dict(Counter(labels))
+        labels_x = range(0,k_component)
+        labels_y = np.zeros(k_component)
+        for i in range(0,300):
+            if i in labels_dict:
+                labels_y[i] = labels_dict[i]
+        plt.plot(labels_x, labels_y)
+        plt.title(bvh_file[-30:-15])
+        plt.show()
+
+        # todo: move it up
+        if bvh_file.__contains__('Ground_Truth'):
+            Hist_GT = labels_y
+
+        hell_dist = hellinger(Hist_GT, labels_y)
+        str_out += ("* " + bvh_file[-30:-15] + "-->" +  str(hell_dist)) + '\n'
+        # str_out += ("* " + os.path.dirname(bvh_file) + "-->" + str(hell_dist)) + '\n'
+
+
+
+        print(str_out)
+
+
+
+
+def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):
+    """ from https://github.com/mseitzer/pytorch-fid/blob/master/fid_score.py """
+    """Numpy implementation of the Frechet Distance.
+    The Frechet distance between two multivariate Gaussians X_1 ~ N(mu_1, C_1)
+    and X_2 ~ N(mu_2, C_2) is
+            d^2 = ||mu_1 - mu_2||^2 + Tr(C_1 + C_2 - 2*sqrt(C_1*C_2)).
+    Stable version by Dougal J. Sutherland.
+    Params:
+    -- mu1   : Numpy array containing the activations of a layer of the
+               inception net (like returned by the function 'get_predictions')
+               for generated samples.
+    -- mu2   : The sample mean over activations, precalculated on an
+               representative data set.
+    -- sigma1: The covariance matrix over activations for generated samples.
+    -- sigma2: The covariance matrix over activations, precalculated on an
+               representative data set.
+    Returns:
+    --   : The Frechet Distance.
+    """
+
+    mu1 = np.atleast_1d(mu1)
+    mu2 = np.atleast_1d(mu2)
+
+    sigma1 = np.atleast_2d(sigma1)
+    sigma2 = np.atleast_2d(sigma2)
+
+    print('&&&&&&', mu1.shape, mu2.shape)
+    assert mu1.shape == mu2.shape, \
+        'Training and test mean vectors have different lengths'
+    assert sigma1.shape == sigma2.shape, \
+        'Training and test covariances have different dimensions'
+
+    diff = mu1 - mu2
+
+    # Product might be almost singular
+    covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
+    if not np.isfinite(covmean).all():
+        msg = ('fid calculation produces singular product; '
+               'adding %s to diagonal of cov estimates') % eps
+        print(msg)
+        offset = np.eye(sigma1.shape[0]) * eps
+        covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset))
+
+    # Numerical error might give slight imaginary component
+    if np.iscomplexobj(covmean):
+        if not np.allclose(np.diagonal(covmean).imag, 0, atol=1e-3):
+            m = np.max(np.abs(covmean.imag))
+            raise ValueError('Imaginary component {}'.format(m))
+        covmean = covmean.real
+
+    tr_covmean = np.trace(covmean)
+
+    return (diff.dot(diff) + np.trace(sigma1) +
+            np.trace(sigma2) - 2 * tr_covmean)
+
+
+def make_unity_scatter(this_seq_latent, this_seq_labels, save_address, pca, MyTSNE):
+
+    normalized_data = pca.transform(this_seq_latent)
+    # TSNE
+    X_embedded = MyTSNE.transform(normalized_data)
+
+
+
+    # str_out = 'Cluster_index, x_TSNE, y_TSNE\n'
+    str_out = '512\n'
+    for i in range(len(this_seq_labels)):
+        str_out += str(this_seq_labels[i]) + ',{:.3f},'.format(X_embedded[i,0]) + '{:.3f}\n'.format(X_embedded[i,1])
+
+    file1 = open(save_address, 'w')
+    file1.write(str_out)
+    file1.flush()
+    file1.close()
+
+def Metrics_analysis(args, km, normalized_data, scalar,
+                     k_component, labels_, save_adress, rewrite):
+
+
+
+    def frechet_distance(samples_A, samples_B):
+        A_mu = np.mean(samples_A, axis=0)
+        A_sigma = np.cov(samples_A, rowvar=False)
+        B_mu = np.mean(samples_B, axis=0)
+        B_sigma = np.cov(samples_B, rowvar=False)
+        try:
+            frechet_dist = calculate_frechet_distance(A_mu, A_sigma, B_mu, B_sigma)
+        except ValueError:
+            frechet_dist = 1e+10
+        return frechet_dist
+
+    def wasserstein_distance_calc(portion_A, portian_B):
+        # u = [0.5, 0.2, 0.3]
+        # v = [0.5, 0.3, 0.2]
+        # create and array with cardinality 3 (your metric space is 3-dimensional and
+        # where distance between each pair of adjacent elements is 1
+        dists = [i for i in range(len(portion_A))]
+
+        return stats.wasserstein_distance(dists, dists, portion_A, portian_B)
+
+    # frechet distance
+    # frechet_dist = frechet_distance(generated_feats, real_feats)
+
+    # def features_distance():
+    #     # distance between real and generated samples on the latent feature space
+    #     dists = []
+    #     for i in range(real_feats.shape[0]):
+    #         d = np.sum(np.absolute(real_feats[i] - generated_feats[i]))  # MAE
+    #         dists.append(d)
+    #     feat_dist = np.mean(dists)
+
+    import contextlib
+
+    # ____________________________
+    # Train TSNE Transformation
+    pca = PCA(n_components=50)
+    priciple_components = pca.fit(normalized_data)
+    normalized_data = priciple_components.transform(normalized_data)
+    MyTSNE = TSNE(
+        n_components=2,
+        perplexity=30,
+        metric='euclidean',
+        n_jobs=8,
+        verbose=True
+    )
+    MyTSNE = MyTSNE.fit(normalized_data)
+    # ____________________________
+
+
+    # # Plot generated
+    dir_submitted = '/local-scratch/pjomeyaz/rosie_gesture_benchmark/cloned/Clustering/must/Co-Speech_Gesture_Generation/output' \
+               '/IROS_2/BVH_submitted/'
+    conditions = ['Ground_Truthx',
+                  'Cond_BA', 'Cond_BT', 'Cond_SD',
+
+                  'infer_sample_1', 'infer_sample_2', 'infer_sample_3', 'infer_sample_4', 'infer_sample_5'
+
+                                                                                        ,'infer_sample_1r',
+                  'infer_sample_2r', 'infer_sample_3r', 'infer_sample_4r', 'infer_sample_5r',
+                  'infer_sample_6r', 'infer_sample_7r', 'infer_sample_8r', 'infer_sample_9r', 'infer_sample_9r_2',
+                  'infer_sample_10r'
+
+                  # 'infer_sample_2_L', 'infer_sample_3_L', 'infer_sample_4_L', 'infer_sample_5_L',
+                  # 'infer_sample_6_L', 'infer_sample_7_L', 'infer_sample_8_L', 'infer_sample_9_L',
+                  # 'infer_sample_10_L', 'infer_sample_11_L', 'infer_sample_12_L', 'infer_sample_13_L',
+                  # 'infer_sample_16_L', 'infer_sample_16_r0', 'infer_sample_16_r1', 'infer_sample_17_Lr0',
+                  # 'infer_sample_2', 'infer_sample_3', 'infer_sample_4', 'infer_sample_4_2',
+                  # 'infer_sample_5', 'infer_sample_6', 'infer_sample_7', 'infer_sample_7_2', 'infer_sample_7_3',
+                  # 'infer_sample_7_4', 'infer_sample_7_5',
+                  # 'infer_sample_8', 'infer_sample_9', 'infer_sample_10',
+                  # 'infer_sample_11', 'infer_sample_12', 'infer_sample_13',
+                  ]
+    Hist_GT = None
+    str_out = ''
+    str_latex_ablation = ''
+    str_latex = ''
+    GT_latents = None
+    GT_pdf = []
+    sequences = dict()
+    for cond_dir in tqdm(conditions):
+        if cond_dir == 'proposed_10':
+            print("Wait!")
+        bvh_files = sorted(glob.glob(os.path.join(dir_submitted + cond_dir, '*.bvh')))
+        all_latents = []
+        all_labels = []
+        sequences[cond_dir] = dict()
+        for bvh_file in tqdm(bvh_files):
+
+            testcase_id = bvh_file[-7:-4]
+            this_seq_latent = []
+            this_seq_labels = [] # for K-Means algorithm
+
+            with open(os.devnull, "w") as f, contextlib.redirect_stdout(f):
+                latent_org = maake_dataset_for_inference(args.ckpt_path_DAE,
+                                                         args.ckpt_path_Autoencode, bvh_file)
+
+
+            # labels = np.zeros(len(latent_org), dtype=int)
+            for i, item in tqdm(enumerate(latent_org)):
+                # labels[i] = int(latent_org[i]['quantized_indices'])
+                # Store_latent
+                current_rnn = item['latent_rnn']
+                current_rnn = np.hstack(current_rnn)
+                this_seq_latent.append(current_rnn)
+
+
+                # Store labels
+                if 'quantized_indices' in latent_org[0] and False:
+                    current_quantized_label = int(latent_org[i]['quantized_indices'])
+                    this_seq_labels.append(current_quantized_label)
+                    # all_labels.append(current_quantized_label)
+
+            if 'quantized_indices' not in latent_org[0] or True:
+                to_predict = np.array(this_seq_latent)
+                this_seq_labels = km.predict(to_predict)
+
+            make_unity_scatter(np.array(this_seq_latent), this_seq_labels, (bvh_file[:-4] + '_unuryscatter.txt'),
+                               pca, MyTSNE)
+
+
+
+        # all_latents = np.array(all_latents)
+        # to_predict = np.array(to_predict)
+        # all_latents = scalar.transform(all_latents) #normalization
+
+        # labels = km.predict(all_latents)
+
+            # Store for BLEU
+            sequences[cond_dir][testcase_id] = np.array(this_seq_labels)
+            all_latents.extend(this_seq_latent)
+            all_labels.extend(this_seq_labels)
+            # if testcase_id == '002':
+
+
+
+        # all_latents = np.hstack(all_latents)
+        # all_labels = np.vstack(all_labels)
+
+        labels_dict = dict(Counter(all_labels))
+        # k_component = 512
+        labels_x = range(0, k_component)
+        labels_y = np.zeros(k_component)
+        for i in range(0,k_component):
+            if i in labels_dict:
+                labels_y[i] = labels_dict[i]
+        plt.plot(labels_x, labels_y)
+        plt.title(bvh_file[-30:-15])
+        plt.show()
+
+        # todo: move it up
+        if bvh_file.__contains__('Ground_Truth'):
+            Hist_GT = labels_y
+            GT_latents = all_latents
+            GT_pdf = labels_y / np.sum(labels_y)
+            ref_sequences_id = cond_dir
+
+
+        labels_y_pdf = labels_y/np.sum(labels_y)
+        ppl = np.exp( - np.sum(labels_y_pdf*np.log(labels_y_pdf++ 1e-10)   ) )
+
+
+        # 1. Hellinger Distance
+        hell_dist = hellinger(Hist_GT, labels_y)
+        str_out += '\n' + cond_dir + '\n'
+        str_latex_ablation
+        str_out += '\n Perplexity: ' + str(ppl) + '\n'
+        str_out += ('hell_dist' + " --> " + str(hell_dist)) + '\n'
+        # str_out += ("* " + os.path.dirname(bvh_file) + "-->" + str(hell_dist)) + '\n'
+
+        # 2.
+        frech_dist = frechet_distance(GT_latents, all_latents)
+        str_out += 'Frechet Distance --> ' + str(frech_dist) + '\n'
+
+        # 3.
+        wasserstein_distance = wasserstein_distance_calc(GT_pdf, labels_y / np.sum(labels_y))
+        str_out+='wasserstein_distance -> ' + str(wasserstein_distance)
+
+        # 4. BLEU
+        from torchtext.data.metrics import bleu_score
+        from nltk.translate.bleu_score import sentence_bleu
+        from nltk.translate.bleu_score import corpus_bleu
+        pose_sentence_length = 4
+        current_cond_score_list = []
+        for test_case in sequences[cond_dir]:
+            predicted_corpus = sequences[cond_dir][test_case]
+            references_corpus = sequences[ref_sequences_id][test_case]
+
+            predicted_sentence_all = []
+            references_sentence_all = []
+
+
+
+
+            predicted_sentence = predicted_corpus[:] #[word_i: word_i + pose_sentence_length]
+            predicted_sentence = predicted_sentence.tolist()
+            predicted_sentence = list(map(str, predicted_sentence))
+            # predicted_sentence_all.append(predicted_sentence)
+            predicted_sentence = [predicted_sentence]
+
+
+            references_sentence = references_corpus[:] #[word_i: word_i + pose_sentence_length]
+            references_sentence = references_sentence.tolist()
+            references_sentence = list(map(str, references_sentence))
+            # references_sentence_all.append((references_sentence))
+            references_sentence = [[references_sentence]]
+
+            score = bleu_score(predicted_sentence, references_sentence, max_n=4,
+                               weights=[0.25, 0.25, 0.25, 0.25])
+            # score = sentence_bleu(references_sentence, predicted_sentence)
+            current_cond_score_list.append(score)
+
+
+        avg_BLEU_Score = np.mean(current_cond_score_list)
+        std_BLEU_Score = np.var(current_cond_score_list)
+        str_out += "\n BLEU: "+ str(np.sum(current_cond_score_list))+" AVG_BLEU Score:" + str(avg_BLEU_Score) + " +-" + str(std_BLEU_Score) + "\n\n\n"
+        print(str_out)
+        str_latex += cond_dir.replace('_', '-') + ' & ' + '${:.3f}$'.format(frech_dist) + ' & ' + '${:.3f}$'.format(hell_dist) + \
+        ' & ' + '${:.2f}$'.format(ppl) + ' & ' + '${:.3f}$'.format(avg_BLEU_Score) + ' \\\\ \n'
+
+
+
+
+    metric_file = open(save_result_path + "/Metrics.txt", 'w')
+    metric_file.write(str_out + '\n\nLatex\n' + str_latex)
+    metric_file.flush()
+    metric_file.close()
+    exit()
+
+
+def normalize(hist):
+    return hist / np.sum(hist)
+
+
+def hellinger(hist1, hist2):
+    """Compute Hellinger distance between two histograms
+
+      Args:
+          hist1:        first histogram
+          hist2:        second histogram of the same size as hist1
+
+      Returns:
+          float:        Hellinger distance between hist1 and hist2
+    """
+
+    return np.sqrt(1.0 - np.sum(np.sqrt(normalize(hist1) * normalize(hist2))))
+
+def FGD():
+    pass
+
+# Todo: Remove this from here???
+# Todo:-------------------------------------------------------
+from scipy.special import gammaln
+from scipy import linalg
+import numpy as np
+from numpy.matlib import repmat
+# from numpy.linalg import inv, linalg
+
+
+def mapdp_nw(X, N0, m0, a0, c0, B0, epsilon=1e-6, maxiter=100, fDebug=False):
+    '''
+    MAP-DP for Normal-Wishart data.
+
+    Inputs:  X  - DxN matrix of data
+             N0 - prior count (DP concentration parameter)
+             m0 - cluster prior mean
+             a0 - cluster prior scale
+             c0 - cluster prior degrees of freedom
+             B0 - cluster prior precision (inverse covariance)
+             epsilon - convergence tolerance
+             maxiter - maximum number of iterations, overrides threshold calculation
+             fDebug - extra verbose input of algorithm operation
+
+    Outputs: mu - cluster centroids
+             z  - data point cluster assignments
+             K  - number of clusters
+             E  - objective function value for each iteration
+
+    CC BY-SA 3.0 Attribution-Sharealike 3.0, Max A. Little. If you use this
+    code in your research, please cite:
+    Yordan P. Raykov, Alexis Boukouvalas, Fahd Baig, Max A. Little (2016)
+    "What to do when K-means clustering fails: a simple yet principled alternative algorithm",
+    PLoS One, (11)9:e0162259
+    This implementation follows the description in that paper.
+    '''
+
+    # Initialization (Alg. 3 line 1)
+    (D, N) = X.shape
+    assert (D > 0)
+    assert (N > 0)
+    K = 1
+    z = np.zeros((N), dtype=int)  # everybody assigned to first cluster
+    Enew = np.inf
+    dE = np.inf
+    ic = 0  # iteration coung
+    E = list()
+    # Convergence test (Alg. 3 line 14 and Appendix B)
+    while (abs(dE) > epsilon and ic < maxiter):
+        print("Iteration:", ic)
+        Eold = Enew
+        dik = np.ones((N, 1)) * np.inf
+        for i in range(N):
+            dk = np.ones((K + 1, 1)) * np.inf
+            f = np.empty(K + 1)
+            Nki = np.ones((K), dtype=int)
+            xi = np.atleast_2d(X[:, i]).T  # current data point
+            for k in range(K):
+                zki = (z == k)
+                zki[i] = False
+                Nki[k] = zki.sum()
+                # Updates meaningless for Nki=0
+                if (Nki[k] == 0):
+                    continue
+                # Update NW cluster hyper parameters (Alg. 3 line 7)
+                mki, aki, cki, Bki = nwupd(Nki[k], X[:, zki], m0, a0, c0, B0)
+
+                # Compute Student-t NLL, existing clusters (Alg. 3 line 8)
+                dk[k] = stnll(xi, mki, aki, cki, Bki, D)
+                # Avoid reinforcement effect at initialization (Appendix B)
+                if (ic == 0):
+                    Nki[0] = 1
+                f[k] = dk[k] - np.log(Nki[k])
+            # Compute Student-t NLL, new cluster (Alg. 3 line 9)
+            dk[K] = stnll(xi, m0, a0, c0, B0, D)
+            f[K] = dk[K] - np.log(N0)
+            # Compute MAP assignment (Alg. 3 line 10)
+            if (fDebug):
+                print(i, 'Compute MAP assignment K=', K, 'f=', f, 'dk=', dk)
+
+            z[i] = np.argmin(f)
+            dik[i] = f[z[i]]
+            # Create new cluster if required (Alg. 3 line 11-12)
+            if (z[i] == K):
+                K = K + 1
+        # Remove any empty clusters and re-assign (Appendix B)
+        Knz = 0
+        for k in range(K):
+            i = (z == k)
+            Nk = i.sum()
+            if (Nk > 0):
+                z[i] = Knz
+                Knz = Knz + 1
+        K = Knz
+        Nk, _ = np.histogram(z, range(K + 1))
+        # Compute updated NLL (Alg. 3 line 13)
+        Enew = dik.sum() - K * np.log(N0) - np.sum(gammaln(Nk))
+        dE = Eold - Enew
+        ic += 1
+        E.append(Enew)
+        print('Iteration %d: K=%d, E=%f, dE=%f\n' % (ic, K, Enew, dE))
+
+    # Compute cluster centroids (Appendix D)
+    mu = np.ones((D, K))
+    for k in range(K):
+        xk = X[:, z == k]
+        mu[:, k] = xk.mean(1)
+    return mu, z, K, E
+
+
+def stnll(x, m, a, c, B, D):
+    '''
+    Compute Student-t negative log likelihood (Appendix A, eqn. (20))
+    '''
+    mu = m
+    nu = a - D + 1
+    Lambda = c * float(nu) / (c + 1) * B
+    S = np.dot(np.dot((x - mu).T, Lambda), (x - mu))
+    _, logdetL = slogdet(Lambda)
+    return float(nu + D) / 2. * np.log(1. + S / float(nu)) \
+           - 0.5 * logdetL + gammaln(nu / 2.) \
+           - gammaln((float(nu) + D) / 2.) + D / 2. * np.log(float(nu) * np.pi)
+
+
+def nwupd(Nki, xki, m0, a0, c0, B0):
+    '''
+    Update Normal-Wishart hyper parameters (Appendix A, eqns. (18-19))
+    '''
+    xmki = xki.mean(1)[:, None]
+    xmcki = xki - repmat(xmki, 1, Nki)
+    Ski = np.dot(xmcki, xmcki.T)
+    cki = c0 + Nki
+    mki = (c0 * m0 + Nki * xmki) / cki
+    xm0cki = xmki - m0
+    Bki = inv(inv(B0) + Ski + c0 * Nki / cki * np.dot(xm0cki, xm0cki.T))
+    aki = a0 + Nki
+    return mki, aki, cki, Bki
+
+
+# Todo:-------------------------------------------------------
+
+def test_mapdp():
+    d = pd.read_csv('toydata.csv')
+    N = d.shape[0]
+    X = d[['X_1', 'X_2']].values.T
+    Z = d['Z'].values
+    # Set up Normal-Wishart MAP-DP prior parameters
+    N0 = 0.5  # Prior count (concentration parameter)
+    m0 = X.mean(1)[:, None]  # Normal-Wishart prior mean
+    a0 = 10  # Normal-Wishart prior scale
+    c0 = 10 / float(N)  # Normal-Wishart prior degrees of freedom
+    B0 = np.diag(1 / (0.05 * X.var(1)))  # Normal-Wishart prior precision
+    # # Run MAPDP to convergence
+    mu, z, K, E = mapdp_nw(X, N0, m0, a0, c0, B0)
+    # # Plotting
+    plt.figure(figsize=(10, 5))
+    for j in range(1, K + 1):
+        i = (Z == j);
+        plt.plot(X[0, i], X[1, i], '.')
+    plt.title('Ground truth')
+
+    plt.figure(figsize=(10, 5))
+    for j in range(K):
+        i = (z == j);
+        plt.plot(X[0, i], X[1, i], '.')
+    plt.title('MAPDP estimated clusters')
+
+    plt.figure(figsize=(10, 5))
+    plt.plot(E, '-b')
+    plt.title('MAPDP objective function')
+    plt.show()
+    exit()
+
+
+if __name__ == '__main__':
+
+    '''
+   ../output/DAE_old/train_DAE_H41/rep_learning_DAE_H41_checkpoint_020.bin
+../output/autoencoder/toturial/ICLR_VQ_DVAE/VQ-DVAE_ablation1_checkpoint_020.bin
+
+
+ ../output/GENEA/DAE/train_DAE_H45/DAE_H45_checkpoint_020.bin
+../output/GENEA/VQ-VAE/VQVAE_checkpoint_020.bin
+    '''
+
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("ckpt_path_DAE", type=Path)
+    parser.add_argument("ckpt_path_Autoencode", type=Path)
+    # parser.add_argument("pre_processed_pickle", type=Path)
+    # parser.add_argument("save_result_path", type=str)
+    args = parser.parse_args()
+
+    args.save_result_path = os.path.dirname(args.ckpt_path_Autoencode) + '/clusters'
+    if not os.path.exists(args.save_result_path):
+        os.makedirs(args.save_result_path)
+
+    args.pre_processed_pickle = args.save_result_path + '/org_latent_clustering_data.bin'
+
+    if not os.path.exists(args.pre_processed_pickle):
+        maake_dataset(args.ckpt_path_DAE, args.ckpt_path_Autoencode,
+                      args.save_result_path + '/org_latent_clustering_data.bin')
+
+    # if Flag_VQ:
+    #     make_VQ_Centers(args.ckpt_path_DAE, args.ckpt_path_Autoencode,
+    #                       args.save_result_path)
+
+    loaded = pickle.load(open(args.pre_processed_pickle,'rb'))
+    liaded_len = len(loaded)
+    loaded = np.hstack(loaded)
+    print('len loaded', len(loaded))
+    print("Loaded successfully")
+    eps = 3
+    min_samples = 5
+    save_result_path = args.save_result_path
+    clustered = cluster(loaded, eps, min_samples, save_result_path)
\ No newline at end of file
diff --git a/scripts/creat_human-study.py b/scripts/creat_human-study.py
new file mode 100644
index 0000000..f23a505
--- /dev/null
+++ b/scripts/creat_human-study.py
@@ -0,0 +1,126 @@
+import argparse
+import math
+import pickle
+import pprint
+import random
+from pathlib import Path
+from datetime import datetime
+
+import librosa
+import numpy as np
+import time
+
+import torch
+from scipy.signal import savgol_filter
+import joblib as jl
+
+import utils
+from pymo.preprocessing import *
+from pymo.viz_tools import *
+from pymo.writers import *
+from utils.data_utils import SubtitleWrapper, normalize_string
+from utils.train_utils import set_logger
+
+from data_loader.data_preprocessor import DataPreprocessor
+
+from inference_Autoencoder import smoothing_function
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+
+
+def mk_hs_slots(tags_list, name, transcript_path, dict_, result):
+    print(name)
+    transcript = SubtitleWrapper(transcript_path).get()
+
+    word_list = []
+    for wi in range(len(transcript)):
+        word_s = float(transcript[wi]['start_time'][:-1])
+        word_e = float(transcript[wi]['end_time'][:-1])
+        word = transcript[wi]['word']
+
+        word = normalize_string(word)
+        if len(word) > 0:
+            word_list.append([word, word_s, word_e])
+
+    q = 0
+    for word in word_list:
+        if word[0] == 'like':
+            q = q + 1
+
+    for tag in tags_list:
+        for word in word_list:
+            if word[0] == str.lower(tag):
+                dict_[tag] += 1
+                result.append((name, word))
+
+
+
+
+    # print("Wait")
+    return dict_, result
+
+if __name__ == '__main__':
+
+    '''
+        ../output/text2embedding/text2embedding_checkpoint_100.bin
+        /local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts/TestSeq001.json
+        ../output/DAE/train_DAE_H41/rep_learning_DAE_H41_checkpoint_020.bin
+        ../output/autoencoder/VAE+sim/Dautoencoder_fxw_zinput_VAE_checkpoint_020.bin
+
+
+       ../output/text2embedding/text2embedding_300d/text2embedding_300d_checkpoint_160.bin
+/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts/TestSeq001.json
+../output/DAE/train_DAE_H41/rep_learning_DAE_H41_checkpoint_020.bin
+../output/autoencoder/VAE+sim/Decoder_trained/autoencoder_decoder_trained_checkpoint_020.bin
+
+
+              ../output/text2embedding_ABST/text2embedding_ABST_checkpoint_100.bin
+/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts/TestSeq001.json
+../output/DAE/train_DAE_H41/rep_learning_DAE_H41_checkpoint_020.bin
+../output/autoencoder/autoencoder_both_traned_sametime/autoencoder_both_traned_sametime_checkpoint_100.bin
+
+       '''
+    random.seed(datetime.now())
+    parser = argparse.ArgumentParser()
+    parser.add_argument("ckpt_path", type=Path)
+    parser.add_argument("transcript_path", type=Path)
+    parser.add_argument("rep_learning_checkpoint", type=Path)
+    parser.add_argument("autoencoder_checkpoint", type=Path)
+
+    args = parser.parse_args()
+
+    pre_transcript_path = "/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts/"
+
+
+    # Read softbank tags
+    file = open("../output/hs_testcases/softbank_tags.txt", 'r')
+    tags_list = []
+    for line in file:
+        seprated = line.split(',')
+        tags_list.append(str.lower(seprated[0]))
+
+    dict_ = dict()
+    result = []
+    for tag in tags_list:
+        dict_[tag] = 0
+    for i in reversed(range(1, 11)):
+        transcript_path = pre_transcript_path + "TestSeq" + str(i).zfill(3) + ".json"
+        transcript_path = Path(transcript_path)
+
+        dict_, result  = mk_hs_slots(tags_list, "TestSeq" + str(i).zfill(3), transcript_path, dict_, result)
+    total_categiry = 0
+    total = 0
+    for key in dict_:
+        if dict_[key]>0:
+            total_categiry+=1
+            total += dict_[key]
+            print(key, ",", dict_[key])
+
+    file = open("../output/hs_testcases/word2slot.txt", 'w')
+    for item in result:
+        str1 = item[0] + "," + item[1][0] + "," + str(item[1][1]) + "," + str(item[1][2])
+        file.write(str1 + '\n')
+    file.flush()
+    file.close()
+    print("done!", total, "of", dict_.__len__())
\ No newline at end of file
diff --git a/scripts/data_loader/__pycache__/data_preprocessor.cpython-37.pyc b/scripts/data_loader/__pycache__/data_preprocessor.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e30b294d83d4be1236ed47ebf19677beb7917239
GIT binary patch
literal 7591
zcmb7J-EZ7hcIVfS9L|SEqwf#fl9Sk$V_V85-RuT|cau1BHwD~m?5?v+9G05PkvN(m
zN0%H$o+Z%&%A!fJM%>qSQH=Jjz+#_^_77;E^3;dLeJD^AC}5y}LH42joy!?nu@e-X
z0q-~OJ@=e*&-vZMw`(;=!S7%G@P{4eM~d>FRGIt?6n=~+`3(}LxC&F5)>U0q=9;U?
zTz3tbm)sI^y=(R?*HVkIwrk72vRjt9<2uNVZlzaoE8{U$w<`N;ZcXNOw~oBjo$58*
zhN=YiXgaVv&C$$>!vC6?k;beO)t$brFq@S>Q&{;}b7z@$SJ`leb89Ug_-U}l{M27d
z{9eBsBx_GVzH(<n6_r~-(7zk(_JSxy)kqIm54rt**blm46b#MQfcbB13BA=HwCJtL
zPe<X$c#^k~1PWN7VEa={WBO;Rt22X@K2yL5lUc}1fmslDthyF+SOsl1tFjvMGOL5s
zhG^UZiSP5E&*N5*Br)G^*_fMtB#|<nckm?tjfAPG;;EUDDV~;<km{L<)W|HPrPS<L
znMR~A<B^UQJEOFmDII&{oG4$;=5}U}DklmnEh*@!-iI`Qs(he)sH8QF((o873NvSw
zV-@}NOzlh|Zy=vW-b6mbtbLvTeLCBj8_i>UBVB+TXq`pWO8g&Bdz$F6WzP~T=jE)-
zoIg?WvZPW+t?~%#dCRPdbOkf7B41^+pL0wBYif6fuO|D~Z{O`4{p{9zKm19rGknG0
z-un3u_ujep>f7C&{q6UX@Bit&?cJm9*1^F4-Ve6o;Nj0-ySKfe2^-R9anyY%jBec8
z7ZwD~!ypO&@}{B){nlIWin>2Y;~;9qEa2YZL8$zzqCA-elSvVJe&0*NVZc>tI4{h&
zA4GoWT7EzD_JfDQioAZD1c_)SgB=zghDjJlUV=qfAQqDSpc|xU;DOY=FxvIn{Y2Cg
z(1A4ruNxc&T@cq}JHnvJMK$3qkLq3!9g1o^NQ)W^xhU-fyI~}ZB=u9*j=V#x5+_0r
zqg2fJ0~W?!q2ri0+D-Sw0`v)GpGUNGYcFW+_hXE9Ybc~%-%s~EV#54cCY%{I`*he_
zr)&~N<o5zm8l+)25tWqtVdNE+rEcuAGwXPM#JnC@+Z84Y4#QR;rm&vZ=9po;`6iEu
zL4EKDT=7#dP*hnm-Zn9z7J0;m@$nUNr`(%NBlp@M26zcX)(t$4L%-pGQ$e>a44>~N
z*m5@nGl;X`C7*{x`4lA$Bw~id_>7!*lX~>@DB?Ag{z2i_kbV8DP3V%}y`HqNN^5Vk
z--&y{Lw~r*<0K5c-5^N^Jn(iP;Ju#D_cvSJI11S2JKaH&1{{*u?4h*zPJI1tKL}cT
z-tF-#w}U9aoZ!GFCJ*~*vPp)^Bd!L#D1%R@CgMi_p|CwKjKb9Ora@b>g+x)!(|;4C
zs=BDnYqn}@4r+Ccdh5EO+G;~J(LP*y*7V$%%-JHnryuSA)C37saLAR8H2ja$T+P&x
z26Lh>DY=HSexl?sKDlwUml~OVtlm-{eEhMNmvS>Lu~G)R2m@sumsBM+M^<K@sJWF{
z%*OjR@-p)B(UsKB%JSV+yzAtZOc|9kJFA>%s)9M31tm8Qy7!fXt@}ieGg4vP7V|3B
zsASa%MaLNZ)W8wlWcn}zy-__a$u?<ls&}?=_Di4g5YnPe%hNwJ<!M_JTHF?@BXm?n
ziF7J?LR@m|ccpvrTJZpfn{U+k9KE|h$s!T~XTo9EIr$Dz>O&DyVf7#SoX4Mv3WNId
z(C@<9h;k4OdZY<~n4X;IHjjHAEyrJ!??b7Q01EyvYGJ7>Xmc$x9ne;s$vE}91tt8C
zXt*_|O6X*@N{4=TfD<ft#J`6D{8gI3mPaIHN5YURy-Yo%^F?wUC86^{M1zxyNEEfM
zR-r<M+JrJWP$?Uws@ia>>Z<ObWjOnsa$$lN@qm7$ucvzTd$d9?R7hVZ2Zr20PFhvk
zRdN%Lg~!HI#^d0r;Hlyvx25n;kdnjFQaGZC8$+oyc4TI1jqJ2c8cF?&QU_L)6F?B?
zQXOp-R$5Rpbpg%~EmLW=QyV?kexmY!N~cDRbUK@Y7HiPnMr1H0Z#-6yew#ManXG=S
zlIA{EAN*}Toxy2#oLMDrW=&cbGta;!H?x^v(jI4JD;#b<m(8G`O5{>!;o9f2d1gP-
z^94}7aICWOKIY_C4!)wDWX)nPD++SsSe?aA^2LlCFj2n*fUuY?$u*Y2qZK@>M_o{d
zemtdRxObe)5A$=`a(0edW|iz5@uDcpH_xYYo%zl}cAnap^GM4tpk*<;kS=wW$$94&
zvX#+FcHu<DnOYsKuqu@vs|OsXQl(wWUDSSE%GWXrBWLAE8>IzVDx<U_OAbn_vSgyP
zCQBsOuUI`_KRTZ+fZF-|Vz$nvvWu*`q%0_}EBX3k?O=^8z-TpF&l)FM{z6v6`!C4%
zt0=9@QUxVtbdJ`i`oULtr<&DR<JVgLBIJ2Kdl3?SMhDn5<O>UuUSQ2*yai5MkVAFf
zc&zh(!|1inI-6m$kJQKNmyP@-v?%EL@aR|RMc67UbFwnPStY9-TcAGu*vc>Em-8$6
zMt(JaIlq=Jjb6wuLvEK*dNI3((ls_WqIoWzsH4kl9xd0iOWEb@O16<*&0fx~?OOSb
z^vY->yMYl`VQX(>FO6Q#uA@$tQ8Dl?GhSCT#k&OF{QG?+;z#|0p7dH~9vh(KlaCEa
z$!31_DGlGT@@95**Z6;|sUPg!KVx&HRb99Pn@3Q|Ef+|I`=7$d#+)VmHCO?mdPC#J
zZa)<o-`UVbnYz4gn54o1-r=d2hCm&507OBhCv+PD#8V#%E#WZOIuF_fPAf^1=GKeX
z3Xl-RZk1l0)c8;EI)9HAR=!r(FwyiFI!Tg>g=cTSf6sf>d$+d}Fh&SUlB0dPGi;o;
zt=;tx1Gau|L*<*8ZD{`DjjdO<a#8Y=Rv2zvmmq>)qSYkYSfUd7YJQoLWhBDr0f>oG
zyBo(`=)i6glSqgHcasQ%fSNFaBXA=TW<PB0cVYVp7I1>9W0%7HXcwq+H%Pr=6ToG}
zDxe|iu)5(6Ac`-nHts}#C~mzMbd!G2N_o5sMCMld@uxt+Y3u=xYEfV4J^x6UejmV;
ziE6<OK%Z{F#S|?gK~oog3a-pxgI=G1pU5E(B^{s8hhbk>egY@w!|xQC*^W7O%wM9t
zn`7+#I#tSNd?2XU8MND2O79O+cOsrbc}Nf&Thi?n{)_HmV~96F-ELjFvr}yw*PrT@
z?GKIsp4zcHH<tb6)kI2~|CA=3ANM_7*dw-mTRT)bahFfwDS>;h;7&22TQ696_udD^
z0w2%<Q*id)nd<}i#Td|V;#MT^V;)Y$t3nOI9(fSppk1pC;ZmXTEa@QZ8&{FZ(Q#+b
zwoi_cTRp4xZl&0zTxW7nG4ynl>~<Y604RSD?I*5-O;5_>lZ|yI6UjlsOv2sB@4A&<
zOyE}XoHoNzf^3CBGMO-EtelBkk9RuIO;X1MXN*fU@-8^WZy^!RsbG^yndBmLNcGGe
z;j9qcsb>zfQ1^v;2)(4c(=$kizkyXqK*{frC=%P)fX(=8;|C=Z?Te^a)%sgaby;m{
zhBj{i`Dj(39|GK(YU_1X)0a`J(wpkMx?12Rtx?v{Pk#dE&iFA617qil8MIjoz3LjC
zI^MD|*2EmEG`rSB3lJE_+Ul(8=!CtfKD_uG>Us`Go?<WxGnDYi@WT5j04L~2+nE9{
zaT5p(;T(hs48mhbfyQ)r7Q*tt?7;E+CH~s`3Ot><ukxD*PNt7API_1e9#+Q{-nb(T
zv$XS1G^_w)*`mzme~j#@E#^|Z18Unw;+pcXCgVJKRzja(%mTo7LKig!oOje!a6*$$
zT^n9`jtk#m8(Tya{m$W`a0}tnq(OKT{`})ouH|}e(4J9OW!)Or^^wLTo&dHPmr{KM
z+}bI@JDc#%_PCBPm8_Rjy@Q(#(->|{z;>wPUV}L@8;G<F^yj3`sFFGG{FSUSs-`u(
zS;tHg?PhvsiYfaV|4mlOYZ38#3U?wkuVEg-q&1*ht<zkFOgb|_ky>Z2t)st=8`;!j
z?dS*TJTsn-JJx|ywFhfCLO90?&}uDf;Ex#=6heSs5Q_UoOE%3)Zz|7@8!etFqa{35
zhWp%T`9#4us~88A+kAF>K7(DZe6BF#MEgSd9Ow;EGn>j-`y7z224p-7wEO$NQrfDF
zn%SV-{#@ytXU39ptla{B1!kTbUBG^v(HhMKo`UK&=sr<T2)8+%_0N%xE@pElYFneI
z+oNJV&?;$=d(CD>FJN_5^6bSEeN@e6I^a8yxHX!>%*u&|Tj&Dtwbgm)vBrO!EwV9(
zB~D(-maqexb~Cz!nU}EU<qWL_wXEI-jszyRMpv>W%nNJ?1V^Q1%y<=8aT#IGDd@ur
zdR`_(jM56tQOr<Clhsdj%<{}y*;2k5HJ+si6qpz}mJ9Lr8?tG<NqqV45%p{e(wky7
z-GmPQ3ul=kmD6{&cYiVayW9UU{Om_SO(oy!AtXhae<V-2uwxz~OyzeuSrt($JRlri
zgg{h*^^ALb5S#`V=<r#1X6Vq-jc^yo?!lE@{T!<IRG_k^E<zL&5icr#qa7F^O1mTH
zmG%KdX*afn8UYiQrGN2~;Na;E_G{-&LJzID*T4A@84p_CREDp8tFbtPaBjRQ<37nv
z6ebh!94{02WMS{g|KLZF0~rw$&UoJlZ*5%T7cq{%K*<-B_>_c5M43^9rx*B<u#$s8
z5a9lA%eEwC{B0Uyv|<E$N@>hnd%}@{06+#bX=qxf5d<B5L=(nXN|X??j1yeIunM70
zVX;6)I0d4HUH@Ug5xCn684D1sr1<6oxeg0kDFy0M9}6M2fv}Y*6-$)|(S*7(_7^3@
z7*QV}Xok=f0-tGkh{4I8-w%WtMgVdE?ve~hem_769R!vDED(k@XiZT;P!HZ;65#4T
zLm1L&#ZemW4&p%~EFd77%&nluYquneTxVPXZ`?W}VLk;CDX2CCnjQCzx4LnzfQJMY
z_y#3p*nxuMB=Oo2$p?tO2u@lQUxWy8iHHzimD12@jWTt}Yl|>}z2H9F8V4&QfR{ll
z4cLU|Oqd{e&`sTHkptSW;D{e!O}C234+loZ5-m(nC?x`Uq?oQVVMA~rYLcI)>@--|
zZoAXpSkNO1r^bS&xF}eNi1MVN5UQw;nc$7<W;fnF^tn)n_%>3gg4Yf?qO8S$;^e6O
zm&7NE^Y8%41|5$!3X0y4@g1i`c>^fN(dw#U0^ewkxhC<5iPm}b3Vd%9$c504sh-Dw
z(XdO)z&8!_G%(LRAs0CtjcRD7*3d7i7lDn2YyZ~~JjE;~XaaEsX8Cgzn0i-va20VQ
z_&$kLfK%WjjTG^tOr;ng+Tk@awBJ<@jQa|_u+b^Oz?7JYC<fsiYY88b@KF4*gg2e>
zn<bc-%Gkuvo#5>CuZ?gYdxH~D94@56QF=WIdtui{^zh+zpZV9fwq7ASSs-29c3>KD
zccT+tn9Fi7xn2%0uRs5LdeP(o%mX6G?2Qdm9@yf1%Nti6%Zu{qSi~CTcP|X2$RJOV
zxXt3;2)QFbUfe9CdeTvoGxL9@vEPjMafY_fNDt_uNf=9kv?<ot&<dRxo_kJeXD^EQ
z&bEz6Ji^rU)xh(F<I$lTfCf)gJrAF3x<wDaLCe2G2|;gs38Ak6Eu8PYm~p~-QYj@x
zXQ-6)N|hz0Xy8eR8@B3Pv@h6yp=?*1_D7uhHs*xoc`R;u9w%SJ|BRBqqr|6#uAnXg
zu|Yp6z6o-24g5<=JW9?}LeP-Y3j8BVK1AYLWD@tolpEBvjs!*<pAJPeOg#V4hd%7!
e(o95r8|A`&vtY?h(&>bjq40e1uRfPAs{ac?#Z68C

literal 0
HcmV?d00001

diff --git a/scripts/data_loader/__pycache__/lmdb_data_loader.cpython-37.pyc b/scripts/data_loader/__pycache__/lmdb_data_loader.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..932275756bd8e126931372b3449f1f23f877dfe7
GIT binary patch
literal 19298
zcmeHvTaX;rd0t=U(sSLJ**k6;f+WZhfdvJ{v_&BlD1t;}Nf1RKrZgeb8cg>tW_B)%
z)4kwg*dxgTC4qJ*(Rapi?Ac1H;!BmS2dA98rJTf8s$!=~aURldQBq0qHF-!%PF3=K
zf6sjvH<8LycB@aH(|zv!pa1fm|IDXzxwMAAKmPB3eeExHHSIq$G5U)kasik3zYsW@
zqqnqbbl0o8q>ZW}ce5Ikd%PN#d!m}a-DoA-R@Ks3#%!h9>1tZyu~w#?t!5=2Z{^zg
zYF^@cYoWRz<qOpU?uk~hU8<Ib?-r|zZep<HCYsBGm3>Y9x|8e}j<v5>mz<Q7eoL#a
zUecV5lYL8bvTpjeSzSXq=j4&jOL`sY1*d>?LDEN%E;=QoOOif{^rEwb^pd2HIr<f?
zviv?quU)R_VRonH*q(c1*X`6@q>~qGe$8`zL>C-)v$osv?RvM>s`-3d;M<p#yQ8{w
z*YmoH&#pGw?iE)xT<^1=cWWK5m#FVLwP&9W&HB!6U1v-9OX0F`nYg?kAPBT3rr$Bp
zHPeZ`V_-_mz8>h_X<rYt?`W^Z`bOXE$KKXDE5j$r(UX3>pXeujV_-UQC-IgR7#FqM
z`o7+`{8(V1zT`W`Ail5l4L=c>xAne>dkptjpsQa-JdSuA_XO?<+>^K`zpZWRj^(7@
zvhHY3dR=3kZ)+&=F9IX50yBu+(Smq@+?&=Fv^jdV8NY&$glWz8o-Yq4tGv0}sr!v?
zr@Yx!<yN=mG&)=5H@eCxk7nB|H#+5;HPxtHYq{l?+u8E3d*$(japE{xS2$r=1U{zB
z(bv%_G-X^?%z*hfQ4%+O@AouCy8qEHo^5sOwbmK0u4;b$`q`akx9#4l_0FoU*KqAE
z*YkIkYhS|`zuvB??X&e(x8pizUuf-mKE7jT>uejbvoCbdT-kBm`gQx#u*fC1<DyQC
z>MSa6?D*bU2Lxerc&<9zYCG5LsRtW7w?gZDtJc2e)Sg>K*Zv1Ct@k)*fiyQFEPJ!F
zabEiDJ%>n?|7yhu&29Hqm{jgfS9$JuVhmKmWh@i3gvb^q=Jjk}#{}uO^<Oe>#I8hB
zQPCAi4mBAYPb*TO%up0$ko-~le5>L4^=^CTxl>4ctUICgp14mJM`bYYnQNS6d$XZX
z&xqK0PvBZ)osS`Cu#3mu#y1oq!*>Lc=_Ih*`W@|8q{|g!Tz)h}DL*~L0J+z-sUgZ(
zDuybckx+Lq;yhxL5vn4RAHy{o+PQlV&1ux5>A;7Wj8bvFQ8!z`n629Gp55qdc1O((
z=W<+ly21O*<+zN{^W{o1jDfHCq1kKfgmJ&C>es{c=enJ(t8ND`6sIPPfnv9!PNl*a
zNv@jn+>TEsX172up*0$2HMP_AT#xh?#%sGyqZ=;IWVOlLI=&Y!)T0Gp@1T9qc$mNR
z^3{*qFSf6_4p(nA$2vXKFDo^gd0p}IC{94QIBIi>u=&<Ii9j<;@t7XdOU6oKNiP_d
zZXw0cGFOb4zGSFJkgLwDqd$)I0xs_~0#~Def^ZU^SvAQuAs}KA2ZWGHIu_CiCk6J9
z4)a&Bk{kXla%}MPIu`=}*i){A7jSw176JH}W^3SXnym*hga&w=W}6O3){gn{0l4J8
zK1l8}ZAm&gPN)3zAmiwRY+&qz|LUmCF;+G3;aoF6SO_4>yl?o0Kp%0-ATHcK*-r(j
zL2+NxwN63mh@%eVW*vU1xfobZLijlQwFFtQG+5r(zPr><2Wg5*$69B9R$ql6{-XAZ
z_9fgg?$qVp(kpJnY)Ka7C*JHov9YlcChU5xe%*Zu)SI9A{H<R3rS5LWDeoxvj4(2A
z=vsTH<$C2!)oqvG?2{E)3bl4Sty`qM`gWKE$x!4~)<P@ddkqyPJD?p9TUhdTuQ`pI
z4G&~wd&uax;i|Xowp<?%lq(H@r0mTdFHE;;h_$<p+X{^ih`y#&4SdgoP<DH6m=c!S
zhG-2fiF&>hCR^RDEz}UkxDLYvq)fZ!hep>6V>>nfdYEwcuzo!i$0rU=wcA0*!Jore
zcgICV(r34GGmKwzQ4bcKkK!rxHX3^+gGX*_GmO>LmZzRXEvmxcqpWX{!<-w3dYZ4f
z^1~eFGMY-;!L($fUQ5L)H2u9!#GfdwqLpLWU?&aVwx7kLXt@{(ee_?zaw%Y;Wc4&J
zqhOYB@2$>K_{OM?F&8Bl<|U*~c1)E6M506SQO_Zb+oy1w?@iL?8n^9N@J`sKruSA3
zG$vW<T%!D=fF3Wa?;*jqnmQNjJNkE@=|d?Q7y-x|DoYHCiWwM#SYRR+Eipce1u-Z+
zqT2MWDZ<Xm64(1FKiRZUZwgBt3*a3Siy?t@dQA(`N40(;NFkNM@`VxuJ;Cf}{Olkn
z%Qqh+nhR8FoXjSc>YxDSC=q0+%mmgZ1ZE|7nTsR-hHCh3FY)HHPk-#`K4{UaHyU;7
zA^amTC2@)9ts=rvZecWg&jgTWuWw)shNEq#)RRGsx=LVd$JCQAgS`x()L{l<j<K#q
z%lt<E)#39d<^jt-Hi&aZ28n%sCG+z8{CJU?lMGW=zHr6%yW1e%L~RE&?1cF(*SCe|
zHad>G7p7huWl?J=8777z;|LO!IOrei3<D$+4=q$+`_Np&q!h-Yjq%MqEi6cOG;Bji
zK>oarx;%<l4Fs9imvlp4BQbvb0DYiyn8~(nPjs8Py0+e2`ol~AsrS~Wz+d7uyNzYY
zTWwE$5%tT!-~T`OBf%EcC;99=gTH|Q>$_72%Lq-h6DD_VfoXK#2s4iBL4~We8a-E?
zVZK<s)!30`87)wXl4yF;WN&t7w+%H7N|txMw&RBJ;cXM^*RNz{eS~Ji*;5x;kRuNh
z5sy&En6QKuK)R~q%s_dkuJVORHYL+1x?VJW^V6HLp^pyd>2ITyw}L>^vEs4L)B1{$
zPM7pN*82&wcjWF<JYI%ej=W^alfOXnK)Q)+kY&%|LtT<(FH<2*Xu^pg9DHa5BZhnf
z*+(qk_+0z5IjOmTjdf~m*S5p7O@_YPVmxcxH+E~S=#6?2HL8~wY%<tlFzLg%BkCkx
zd;*t8!3`y0%`&ZH^0bZz<3A4e0xs`!2p(KT5Zf{tS(a5RvM8&m$dat4og&s>DJ;zn
z*uMDui-)Tb-$&a&aW&$rvuecc!~qq;JfK2|a`evKREXrEDui_x6+%6VQ9gjO@Dj-D
zab#7e7(9WXvNk6_)l+;W<i|CtHW-{`FqWa}9Fresz~!!fior7sewqOpvHBSXq!C52
zImICUCX*uIG7+U;;DcEiuAb+!Nqe7WlFKGq=o6Bp!YEV=DCHiq8LJfxaU`3M9weQ^
z<umD57Ha_!N)ufA66~WRlmW)&_c>C`C`Tg-$l)?uR@AQ{k4H;IgSs%!E=axTLL{{*
zSopVqYb>>eil?Ra_xk$Y@A^=Wn}84?-F{P~I3yfoRD3AKA+3f|JUOs}15(^iHPa}c
zfdmKk1BsmyiJgVSPElg-UGxnn=|e`t5(J3x`Z<y5Y2?<vX>3AP_A`#v&*I7jnSLI1
zQLWAdc_;N-=!dS|`1-5bjbpEBQ~CmDE=_5!E@6BlS+4#T5>p~ak>9GHV<2+&Gl)^q
z?WzV8!dfegce)MlR;4f@tT3*?8Qth(2w8rK#lOse+JO2y41}a#V~omRL}C_VVXD?@
zMGCU|EYg*Qh&U-&LcI|guGd(Ef<=+dP018Tay3;O_o@Zh1V)y-33=7n0*KLJ=2@Bz
z3w7x1zH5(L{Y_-@Sa&2~A`z2?aLF66$aGj|u)}(f9zweFHI3y>4wa1wNKh@vo2GUI
z$P?~*(-<m%-|NF(8$js;>BNL|p!`AIA;AEL8YF`_V?i7WAds!T1!d1_Qk_eWNhb|e
z4Wu+w_L4|RTPq-y39*1kvZC~5&{96gK<NXUbF8t_7pa6<d7C}(HXCFED2P{PZK{VQ
zrB^X4&{!(CO^7GrV-HF&RNzF}Qh%2*o55EQRB{o~)DW*)&`WGuG85%Q-IIWXdJZL)
zb?TC$uYKNachxQX#m&t|y#Xta%DhJvQX_jCDkj+&sb&Qfb3#91^iCf_Gk2}9E-VW5
z(IlZZE&Pu%z@aq&vKhdL_BD_T2?jdzD%6-}Y~KWd=s_IjHDFR|$hA$Z+;n;N?|H{5
zU#jhTUZd72a|yd;zgzaNyJebT2Xe!@dv^XIE8Lr{RZ?ewSnlll<(qCDz|3K#1kNM~
z(*|(d*l9IfRXzhu)E&(x`!amzM&KuMm%E$N!Kk<0&S5pnI-l#)nFBR&h|tUCD|p2P
z<k`8~K+lxByZ&Tcr)Ab|%}&?W%Fq3`-~J=>2j{7<$zLTNDcA|{M5o&h6CIl1t<V|{
z@0Jj4B@xjpnM8H*km*q#-h`$LAV3(dJI}^aHOH}KX8s&GMIF@AG5rL@ZSUA2bT`gE
zW3`dygwCi}?49;CK<dCAMXntKJZ=mOScI@ZVX;BYcknha&ue0%3EuqwX>IiJM%4Nf
zqqWH7HB}d%N&swi6EV=6SS_R^U_7dOP0>05w$nLawWh~JaPB`Kk4MR<WlA6!qA0yN
zOGZ?qNr|y=Bh8q&gl<>`EO>CDebPhx@)R!>{Xv+1Qc+7(I}Gm1!ebK_K2^!A^cL&r
zzoHu1`p{zOIG`MZSMS6jL_cLlj=N9Qa;#oT_aDv{6x-~7vJ0$5(|eB`((t?ym{He}
zS2Y+k8NA5gB?dkNnVvTp`znJ?24pEyt6nBTwC-)ZU@4EvF}_{did!qzs+G5jR#EY7
z9oP7e-MoOyyNKX}+k*JYg|K{Z%&ucUzw37+{6UQ^!Xg?aZ3?v<0ZDu?ix3=0v(u1!
z8OPWrM4k0>gFKX|1u9Y0e1SAUG6HHYG>e0hK+W&^i{M+sjX7-cMK<En5N2licmztb
zyuaLB5tv!Wb|G!VRj601gS9ctygr7Rk4&^{^yp|mgYXzi9P^JiPYga1Wc;#|+BaW|
z_p|+6Ki^;I7lPd25huORQJh4*CkKx@(rbYtGM8TpV^`I#dpKJCpr+;lOEWZK<tXYD
zmB;4oy{GW>-Zp1g+^IFx8_*bRw^O?Yv>F)OE>hUs1DNeMu&<Jn^eFk^SgGMEc9db6
zL5(LS4w$~az0+-U{ID?d!fCX__^#h*d11y^wMNH|66?g8r;6DS$U{6m;kY*&bvHEp
zZkS?a5zHQ!;!shAqh8?5JjtNK;G+zlXYe!wF=9T=7%5vcO>7m7`)CW=UYzNuK<0lB
zxx<V+?|L5gIK#Xo#@JHy5~>G4^7hpFm!(g(@xO6-l-JrU7;Yx?>=+QA7~N}lZUHo3
zB8#uwZ{6O<92b5GOJ-t-^T|DNbK!DjEc_9_eDVSMa5#!?a^$DTL#FNV4}_wRvGZ9=
zoY=Wo;(i9`5jMJHgDD4HD2*q1@PhN;G+J|^3Ga<<L%kQI;gFE!lR+WC`hHuxqYa98
zv}S3_&_emQrz~>hUVK+qCuHxJejS#-<YY!=X!pbYAQNN;%fJFMgB7?R5aAyp0N{3T
zj==NPI}FzXn07Sh*zz5X##$x&(AGYs^?0L%f`d#$wwp=xvZMZFADhhqonnR^k(9oM
z^ytaja4VYb2%kYYp<-|)%BxVPTS6`p6VA}?pU5_fi%TP7!It}l2OyR)R6K#yC%e?r
zfy(*^H#on-&L=xvdrQ?ECCeDQeFWRplrj1ia*2i(CK?@J?tYjC<le>haAVsI6W6hy
zhHb*Q+=e*1s<8}(^Mha=2_I|(*nZ4uY;HyqtE*pQVPO!|93JKTokC;^F(;f6OB)vO
zxU8NqvZ<3X3&z3;eaSq`Oq~_xU<%_o9-9vd)X(oGi}HtvfZ&O%!!&&j9x*nk#5Qo5
zxUiYkOn|VHz&+qB1;7hWo5<C)u+Yq-D2qI)L7GpaEw4d_h6kQyvDj|v$kBwLCNhG&
z3rH2zU&_Maez)Zpoj9H?V7Vo*#T6|b;3k7bCxJSa_zpYAV+tKEo>X(0Z;)e!NXsTb
zH*BD>*9#NT?xc`j6#<>Td?-?O_=+JCCM)JNR}3J`nKdKnd#)NRljYGeX+?53S}wzt
zq<$3*%;26v0bD`Ocm?Ub&);_i35;cXjdr62w<G_SJv3qNx}2gtD6$9i(S(?~iv8}Q
zh})4H5v#xs0zzALus0_IfHTEdl^8zANtO|~cqm9XF&Msr7S*pa;PO+y!Qi(Uj8|QR
z2))N=e;)yS&1$~-2TXng!O-dsGdqp?Ha1~48!b1?M?LZPd`T;ZxqpknKV&eJZ=w-!
ze#5o9P1hGV>4GC#$FST=Mpl+v9<mo(R(e@Kg;?(sKlbMsTQFI4gM3+3>i>cWKp`6D
zTGMEnvb`F`xTR{s*X>?!&FC%H95-71grob!6D<PIj=6920cit}z8S;;LwEY}E&z7e
z=aw^6YA3c1*CPPfQ3{*8NTu;yKx%9~8^Lhh4OgFe9^2A@uns=W0A@(wE&2cjK5^qZ
zHi8ZM*KL}SX(_VgcQD}h$#f!nI2G+$#)Zp9)L`=^ZQ}^_o7l_$2+u~Fm+?B>JNz&!
zoEX{~9JM}Holcz_r6T&M#$Zr;p}wv9sHK{%?<xftP)b%!?_)NCb9U8&PA3KCG`BEI
z;7{Iwx3)+sCs2<jJUETb;uwe|t>^J?fml{(k(<3U_a&C;TE_&!mB~v8gal0Dc!=#z
zF+IOF;vq0`&)&Bmvj*>2NUWoA3JU2-4}BuJVebSU5VxVL0AT`}2`xv6Pj*b;k_nVk
zp5CT@19zX>u*$YH0OGMhGI|q$=>xuwuY*t-n@K|eY|(=vfig*4*07D#-dBCN7YVkd
zK%!<qzX#L-9V?2Hth2*79Etill+4}MSzg-1v)lT%slI>XTjG@fxE%TEEs+M<CI@Ni
zV9*QWh~XV#7GgQZZhY%idNQ#8a4MbY?_K-9chXNq`K8Y#^f}2sNA10<-*{H)gl7l(
zkw@)W<j&uQJ?W&DwEhCdUcg)%PWradFYf&?C}IhR{ZddG<Q(+0zZfh!S&VbZ&m%^`
zsrQ#9o|E{B#Pbqg^%n*U5Xo!7lC!XFylbdfP{8O`gB2&n`Nn9OUY?|#M6eL7p2rRs
z?#sjb3TA4_DM$?);~ElC4KpR8oJHi+)z`;$Pv@Mf6SILGFYJGbh<9R6DcUDP%Y~sQ
z%i`tGP}h2C*cRe>>pH!~&(TE{o?W|jIxcVSwpzD@tyFW7`ybS{4h&FHBCGJoqDvqJ
zS_Dv0TMl*gBmMOIF+>$vllnbAOm^G`uyuv4vBuVQXlpHZ(|`Z_cm?nz5s4qVj6su>
zO<Hb*MZ%A}QDJ-sUV6UzeLfX#COqu3Omag2-dY`aro=rU>2Qf;R)GT7#c-`2g`br@
z^u-3V845czg(0ec!tx2W>^c}xgLiI!!X2H$dNRanY5GAl57nawpJ9qTpXh-4$L!x9
zFt~xBx;$c$GgDbD%{|6!RtuwNBSoOPI^7Vm*d0gw->M~XskPyg<En_CdeuzCqNOMN
zO28;0*&buJ>2ji~X6sr@{ZkoyyWu!3SN*Xh>ep-7YMA5JY9H^uK?X?Jj{C;+>H%xU
zPT&zRXL@W}M&8Uqna&z(x)Do5ofZae@;#-0+9)A6-Bq{+E;-@T8SNDjaAkhf3y)9S
z5$0pPbN6Ml2Xw5<mGs@AMH&O@cNzR4gDnOe+0<w$9U{x&Yl!_dE^%W*_a1|1$!Y7j
zwP=}&@9Mb5f0Qm4aQ!$gsPj$$Z(ya^LnqrYY`BcdQ@BKM+`*UlV6Y*8xwNATU?@AC
ze%^t0LlFeQxPZ3>zJ-LRr)!~;aOmq=@Btt?iNWF+TBOzS9vEtJ6?>-y89NqI67!dM
zSVQ2(Kl4|oz{a>ZqyRn?pH|kGLX9Jz*7atdIuv$u(*iagfgW{aaC8hd9s}5z5r>rH
z6N?1>Jt1J@N6^cU_~qs!gOdOoA9b<^z{bZMh#%y79Q8gvIORZ8fKltsC*TA)DVLXW
zdcW8&^%wg~L2)oD_aw?aIVvYM(86W)b$poK>5YxE(FPe#3XF~j(D@TveBRk@I;MIN
zuC}qWt^No_uD*0Qt{E=C6>Q2&yY+xT=K-8k{XAL`oU@v#O|a-nWgf<Riuq16xT~Yh
zZN7br!JjbrXAH=y!?X-d07LZ{pFWgJ&L>&oJcF?}j`}8(-(v7>26q^|&EOXp{2~Lf
zciurPT$vi!6u22Ky>j{T6#gE~d}aB^gfi84*y?x3O~1?7FERLK1l(Ne(AQW1Oqs|q
zc@M<$ugFPXLcux25?m2{5<D;g4x7Nvxn*P{&Ny@!n#L}(Mj|mpF&~@blhYjJZ~zk>
zh*$}gQIIr!VeaGaE4XDEjBbQm&U=8&wK@w~E}<RrFB2E6hN#i8<L}Qru;Xuh<oMf9
zP1e6t7`pzZ$MEII^_R+M<ocVTBN2eX2)={|$AIu<7EcIYmcRufuo8;uG$=WSE#WHc
zq{Tl(U`ymCSh*&c3y+==tjqz|06&9*7$Gh>Y;P<(DYQYbGIaoNnShnxN)cE&><<tg
zY$L9t^&{9a0bUZ_i6uoFC;ei819Rxbdh=-W*f(iiE(8mM<K%A=D>`|2uW*bD$i?lK
zg+X~=8$5#R<UX8>`z5sXXi!3~$B?TunNQfG;Ev)Ty@0Wp{YB)Ic^eerAT1-D%mo~z
z7lC=gK{_b$j8~<2ILtaWg5sb7+Ux&s+@mAJl>}c&s-MM8ysFga82dZ}!BZ|XM)wu$
zAAf=Iml=GK!7B{d*C{n2!{k*2%Ulq7MCKfw=W*J(#W98tCz#URy`JY?(0=lu^ax55
z9Ce}+5lVY%!~p<CfAZdZ=a50p<IqO*bvPoKY%-x!9}YG>4~$J5q(q8|1u+K36wQnX
z82dBC!qjeObgm@nxB_a2RtNg^Eu687d{F-t^Ce+g@Mx9@*$~@kfU~c|LxtYrYD`li
z5h_bl`}r5y9LeUcj##KVI$tGZHiyDhElL-<lvjZy(g>@X?OovMIPL=fi)wmwMymlM
zhpq4%0vesg#dG|W&*04j7hfBCl8cur{L0s$hzlwXrMwI^9!fvd`!xfe=JDPW5BT{%
zTpZ?8BLV~t^U-_tU>1Ub<N19I12G4L*e8Lk^FPH5{d<aRTBWcAo`EHBPVwo7W&_N7
zD2?<#9@1-#jeu1&IW__kn1@4*0KLgQdQ%Xtm>f~VP9B~le0tvUh+1l&X^2Xsc|<Kr
zbFV%-GzoHkUiR(uW)Wh-@=Jq7ADGDG84s_G&4p!@ToH5OJuw$nhUUVG4=(|j3+q$n
z!ixGU<UT@!LFzlocc=>{gOh>*k?I^d^u4^36366ydKCifUEM4B`N8qY6X0X)<*@)F
z6K^m2C%6Fv&=qrnd4Umwo@U-j%I@Vyh9(GmnME&AhQ<KQfTbWMCP+?9kVjAo28Ea)
z+<kl$CI~Pf%;95GB2#ZZ4(v;Bp2G7}{u3|~DnZu&sIwrZ#L(LTLOVZr$|;BmavJrX
z9-JAQAVnx6lX68VC+0_gxxW%Dj>~PJ+{UP!m>{L039>%2JRZ*8=*8(AdRF~Qke8Sm
z;w=w$L3RHAuKf2fJHq_5c7F3l<?Nj0Q9XL_2;kWGQ5V>uhq6t6iY3U>!X%E=@hpk@
z*L*Z<vncZJ!;F=1LF^cu?z;)Zi01>XYjBj5dVZR<#@pMdcm7<#2Q^|8Wlm-FL2Z*T
z=HsNf`ZNde0t1mI-(l=-7E0BkvxOjaf6jd0XCMaOzhP{S;k}4w7r|aUYB(bsg+3Gf
zr*Ci8%tIJp12wW$jD%T^9Wm)0o`;Cw{!ZE|#CiPiE*8t<bBue2-+j0y%dc`ExFF?v
z^M=xVv$HnK47+`A-ex)fz&1;AVzczq{mg`E;3p{fy9@x$qf^NM*hfbfX)6E$!BK{h
z{gRiXi*%{r(Zv)|5NPHLb4M2eUpu+67kfh9!u>)RUOq@;*H?0Ef$hrSMxa1&!YZgA
z2FZT06AP?<5rUx@kntC#G-8sI&r72kXg#1Oi_Ik`FM#v~rUuIa+Jc1w%!EwK&%$+V
zg~z4dG5VzrSR)<Ba55P#V}m2Ofa|a(X@_<Q(4BtiR+Oe^K965wzB>595dMEz>lk0s
z0AVMBBA9wQD8UOq35%ojFc!yaAO&m%V5g6+y%1cw!Y^msM*RnlXFd}h#FXQi&&csi
zxY)trTn=qU;(~`tWb<SCNTy)sk*7s6a!vi`XmrZ(_&_$t?~tPDJE8$lh4%uMn;ft-
zde7fySskL0&zT)FMp_I_h>=E)8fbEYbtQml`+6Qe(B?p+`vLZHvT{Nd(<$;3e))1f
z)s*w8Q0QaPz~{$|4Z`kD)pQZ$<vchJH$#`kX>l>Rs%begDaTId7*uqy`ZGMuIyrLs
z2b>i$GEHWLjHvhG!_NmBovZm!ABV}|Q}rs^yc&INT@8?+6?6Q@wPJ)T%Ur#Vs-|3N
zzQW`NgWqRBnNc-v-UzLmHzLCXCw>)9`RqmF?Jt;#X14lE2LFzM7`Q_@Q{w}X9EaE;
z)wKKugsp0Cc%yjRZF$kbRqoVQbC+J8`qhWXO8Jkh(_uiXBP@<9v2pMo-uZ9HNu@9|
zF;Bc9Y=&{9aQw65<3z55lku*?PYdkP`GRl6?E_Aa-W(x_WAJbA!#{`iCT7erBb~KU
zlu$5KfT*p-xI>ZP{)3*6LrU=^Q!)WFCT|&#WRPn{Z{81YUO0>w&xdQzywC3AqI`Pl
zp7i>m88kmYV=u7Tp+WO944PLIpMq@9;TD~v{1e86ZUhzn1d}ApYDs=?WTVsB5YN&L
zZUAxwVIg`EJ>`y-BHmX$I{$cs?_|$S$<LEik4M{E+=Wx}68m+NpHGVPtt#8U8MW_;
z8|=meXjcio{WgPNK!Eey_;IB$+wkn#%^Li8@Y_N1!y)P<vurcqCXV`31|0bmWr;RK
v%MU-x6CF=}j_{G}?Z{4*{3;3duz=3;;xmLWjIGBDpDcV$vv7U6u=9Tb<1fz3

literal 0
HcmV?d00001

diff --git a/scripts/data_loader/data_preprocessor.py b/scripts/data_loader/data_preprocessor.py
new file mode 100644
index 0000000..d47de75
--- /dev/null
+++ b/scripts/data_loader/data_preprocessor.py
@@ -0,0 +1,402 @@
+""" create data samples """
+import lmdb
+import math
+import numpy as np
+import pyarrow
+from deepsegment import DeepSegment
+import torch
+import librosa
+import utils
+from tqdm import tqdm
+import openai
+import pickle
+import os
+from trankit import Pipeline
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+class DataPreprocessor:
+    def __init__(self, args, clip_lmdb_dir, out_lmdb_dir, n_poses, subdivision_stride, pose_resampling_fps, sentence_level=False):
+        openai.api_key = "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"
+        self.n_poses = n_poses
+        self.subdivision_stride = subdivision_stride
+        self.skeleton_resampling_fps = pose_resampling_fps
+        self.sentence_level = sentence_level
+        self.src_lmdb_env = lmdb.open(clip_lmdb_dir, readonly=True, lock=False)
+        self.out_lmdb_dir = out_lmdb_dir
+        with self.src_lmdb_env.begin() as txn:
+            self.n_videos = txn.stat()['entries']
+
+        self.audio_sample_length = int(self.n_poses / self.skeleton_resampling_fps * 16000)
+
+        self.ckpt_path_DAE = args.rep_learning_checkpoint
+        self.ckpt_path_Autoencode = args.autoencoder_checkpoint
+
+        if args.name != "DAE":
+            self.DAE_frame_level = utils.train_utils.load_checkpoint_and_model(
+                self.ckpt_path_DAE, device,'DAE')
+
+        if self.sentence_level:
+            self.rnn_representation = utils.train_utils.load_checkpoint_and_model(
+                self.ckpt_path_Autoencode, device, 'autoencoder_vq')
+
+        # create db for samples
+        map_size = 1024 * 50  # in MB
+        map_size <<= 20  # in B
+        self.dst_lmdb_env = lmdb.open(out_lmdb_dir, map_size=map_size)
+        self.n_out_samples = 0
+
+        self.sentence_frame_length = args.sentence_frame_length
+        self.audi_smaple_rate = 16000
+
+
+    def run(self):
+        src_txn = self.src_lmdb_env.begin(write=False)
+        total_count = src_txn.stat()['entries']
+        # sampling and normalization
+        cursor = src_txn.cursor()
+        counter = 0
+        for key, value in tqdm(cursor):
+            print("video ", counter, "of", total_count, '\n')
+            video = pyarrow.deserialize(value)
+            vid = video['vid']
+            clips = video['clips']
+            for clip_idx, clip in enumerate(clips):
+                self._sample_from_clip(vid, clip)
+                counter = counter + 1
+            # if counter>5:
+            #     print("Break the chain of loading huge data!!!!!!1")
+            #     break
+
+        # print stats
+
+
+
+
+        with self.dst_lmdb_env.begin() as txn:
+            print("Sample_counter", txn.stat()['entries'])
+
+        # close db
+        self.src_lmdb_env.close()
+        self.dst_lmdb_env.sync()
+        self.dst_lmdb_env.close()
+
+    def _sample_from_clip(self, vid, clip):
+        clip_skeleton = clip['poses']
+        clip_audio_raw = clip['audio_raw']
+        clip_word_list = clip['words']
+
+        # divide
+        aux_info = []
+        sample_skeletons_list = []
+        sample_words_list = []
+
+        sample_audio_list_mels = []
+        sample_audio_list_raws = []
+
+        sentence_leve_latents_list = []
+        GPT_3_STR_list = []
+        GPT_3_Embedding_list = []
+
+        if self.sentence_level:
+            self.n_poses = self.sentence_frame_length
+            self.audio_sample_length = int(self.n_poses / self.skeleton_resampling_fps * self.audi_smaple_rate)
+
+        num_subdivision = math.floor(
+            (len(clip_skeleton) - self.n_poses)
+            / self.subdivision_stride) + 1  # floor((K - (N+M)) / S) + 1
+
+        # Sentence level preparation:
+
+        '''
+        self.sentence_level = False
+        if self.sentence_level:
+            print("Sentence level")
+            
+            str_transcript = ""
+            for i_t in range(len(clip_word_list)):
+                str_transcript = str_transcript + " " + clip_word_list[i_t][0]
+
+            import nltk  # library
+            # nltk.download()
+            sentences = nltk.sent_tokenize(str_transcript)  # whole paragraph break into sentence.
+
+            start_sentence_word_index = 0
+            end_sentence_word_index = 0
+            for sentence in sentences:
+                x = sentence.replace('.', '').strip().split(' ')
+
+                end_sentence_word_index = start_sentence_word_index + len(sentence.replace('.', '').strip().split(' ')) - 1  # to index
+                #     process
+
+                reconstructed_sentence = clip_word_list[start_sentence_word_index:end_sentence_word_index+1]
+
+                start_idx = int( clip_word_list[start_sentence_word_index][1] * self.skeleton_resampling_fps )
+                try:
+                    fin_idx = int( clip_word_list[end_sentence_word_index][2] * self.skeleton_resampling_fps )
+                except:
+
+                    print()
+                # ........................
+                sample_skeletons = clip_skeleton[start_idx:fin_idx]
+                subdivision_start_time = start_idx / self.skeleton_resampling_fps
+                subdivision_end_time = fin_idx / self.skeleton_resampling_fps
+                sample_words = self.get_words_in_time_range(word_list=clip_word_list,
+                                                            start_time=subdivision_start_time,
+                                                            end_time=subdivision_end_time)
+                if len(sample_words) < 2:
+                    continue
+
+                # raw audio
+                audio_start = math.floor(start_idx / len(clip_skeleton) * len(clip_audio_raw))
+                audio_end = audio_start + self.audio_sample_length
+                sample_audio = clip_audio_raw[audio_start:audio_end]
+
+                motion_info = {'vid': vid,
+                               'start_frame_no': start_idx,
+                               'end_frame_no': fin_idx,
+                               'start_time': subdivision_start_time,
+                               'end_time': subdivision_end_time}
+
+                sample_skeletons_list.append(sample_skeletons)
+                sample_words_list.append(sample_words)
+                sample_audio_list.append(sample_audio)
+                aux_info.append(motion_info)
+
+
+
+                start_sentence_word_index = end_sentence_word_index + 1
+                print()
+            '''
+        # Loading cached GP3 requestes
+        address = self.out_lmdb_dir + '_' + vid + '.gpt'
+        if os.path.exists(self.out_lmdb_dir + '_' + vid + '.gpt'):
+            loaded_gpt3 = pickle.load(open(address, 'rb'))
+        else:
+            loaded_gpt3 = None
+
+        for i in tqdm(range(num_subdivision)):
+
+
+
+            start_idx = i * self.subdivision_stride
+            fin_idx = start_idx + self.n_poses
+
+            sample_skeletons = clip_skeleton[start_idx:fin_idx]
+            subdivision_start_time = start_idx / self.skeleton_resampling_fps
+            subdivision_end_time = fin_idx / self.skeleton_resampling_fps
+            sample_words = self.get_words_in_time_range(word_list=clip_word_list,
+                                                        start_time=subdivision_start_time,
+                                                        end_time=subdivision_end_time)
+
+
+            if len(sample_words) < 4:
+                continue
+
+            # raw audio
+            audio_start = math.floor(start_idx / len(clip_skeleton) * len(clip_audio_raw))
+            audio_end = audio_start + self.audio_sample_length
+            sample_audio = clip_audio_raw[audio_start:audio_end]
+
+            mel_chunks = []
+            raw_chunks = []
+            for audio_sub in range(self.audio_sample_length//self.audi_smaple_rate):
+                audio_chunk = sample_audio[audio_sub*self.audi_smaple_rate: (audio_sub+1)*self.audi_smaple_rate]
+                signal = librosa.feature.melspectrogram(y=audio_chunk, sr=self.audi_smaple_rate)
+                signal = librosa.power_to_db(signal, ref=np.max)
+                mel_chunks.append(signal)
+                # raw_chunks.append(audio_chunk)
+                raw_chunks.append(0)
+                # signal = librosa.amplitude_to_db(signal)
+
+
+
+
+
+
+
+            motion_info = {'vid': vid,
+                           'start_frame_no': start_idx,
+                           'end_frame_no': fin_idx,
+                           'start_time': subdivision_start_time,
+                           'end_time': subdivision_end_time}
+
+            sample_skeletons_list.append(sample_skeletons)
+            sample_words_list.append(sample_words)
+
+            sample_audio_list_mels.append(mel_chunks)
+            sample_audio_list_raws.append(raw_chunks)
+
+            aux_info.append(motion_info)
+            if self.sentence_level:
+
+                # For the input side:
+                # GPT-3 Embeddomg
+                str_input = ""
+                for word in sample_words:
+                    str_input += ' ' + word[0]
+
+                GPT_3_features = self.GPT_3_caller(str_input, loaded_gpt3)
+                GPT_3_Embedding_list.append(GPT_3_features)
+                GPT_3_STR_list.append(str_input)
+                # Discretization for the output side:
+                sentence_leve_latents_list.append(self.get_pose_latent(sample_skeletons))
+
+            # if i>100:
+            #     break
+
+
+
+        object_to_save = {'sample_words_list':GPT_3_STR_list,
+                          'GPT_3_Embedding_list': GPT_3_Embedding_list}
+        pickle.dump(object_to_save, open(self.out_lmdb_dir + '_' + vid + '.gpt','wb'))
+        print("Embedding Saved!")
+        if len(sample_skeletons_list) > 0:
+            with self.dst_lmdb_env.begin(write=True) as txn:
+                if self.sentence_level:
+                    for words, poses, audio_raws, audio_mels, aux, sentence_leve_latents , GPT_3_Embedding in \
+                            zip(sample_words_list, sample_skeletons_list,
+                                sample_audio_list_raws, sample_audio_list_mels,
+                                aux_info, sentence_leve_latents_list, GPT_3_Embedding_list):
+                        poses = np.asarray(poses)
+                        GPT_3_Embedding = np.array(GPT_3_Embedding)
+                        # save
+                        k = '{:010}'.format(self.n_out_samples).encode('ascii')
+                        v = [words, poses, audio_raws, audio_mels, aux, sentence_leve_latents, GPT_3_Embedding]
+                        v = pyarrow.serialize(v).to_buffer()
+                        txn.put(k, v)
+                        self.n_out_samples += 1
+                else:
+                    for words, poses, audio, aux in zip(sample_words_list, sample_skeletons_list,
+                                                        sample_audio_list, aux_info):
+                        poses = np.asarray(poses)
+
+                        # save
+                        k = '{:010}'.format(self.n_out_samples).encode('ascii')
+                        v = [words, poses, audio, aux]
+                        v = pyarrow.serialize(v).to_buffer()
+                        txn.put(k, v)
+                        self.n_out_samples += 1
+
+    @staticmethod
+    def get_words_in_time_range(word_list, start_time, end_time):
+        words = []
+
+        for word in word_list:
+            _, word_s, word_e = word[0], word[1], word[2]
+
+            if word_s >= end_time:
+                break
+
+            if word_e <= start_time:
+                continue
+
+            words.append(word)
+
+        return words
+
+
+    def get_pose_latent(self, poses):
+
+        # 1. Load models
+        args, DAE, loss_fn, lang_model, out_dim = self.DAE_frame_level
+        args, rnn, loss_fn, lang_model, out_dim = self.rnn_representation
+        DAE.train(False)
+        rnn.train(False)
+
+
+        # poses, poses_mirror = process_bvh(bvh_file)
+
+        mean = np.array(args.data_mean).squeeze()
+        std = np.array(args.data_std).squeeze()
+        std = np.clip(std, a_min=0.01, a_max=None)
+        out_poses = (np.copy(poses) - mean) / std
+
+        target = torch.from_numpy(out_poses)
+        # target = torch.unsqueeze(target,2)
+        target = target.to(device).float()
+        reconstructed = []
+        # for i in range(len(out_poses)):
+        #     input = torch.unsqueeze(target[i],0)
+        #     current_out = pose_decoder(input)
+        #     reconstructed.append(current_out)
+
+        if DAE.encoder == None:
+            encoded = target
+        else:
+            encoded = DAE.encoder(target)
+        # encoded = torch.squeeze(encoded, 2)
+        # encoded = encoded.to('cpu')
+        # encoded = encoded.detach().numpy()
+        # all_frames_from_rnn = None
+
+        result = np.zeros((len(encoded)//args.n_poses, rnn.decoder.n_layers, args.hidden_size))
+        result_index = 0
+        for i in range(0, len(encoded), args.n_poses):
+            current_dict = dict()
+            input_seq = encoded[i:i + args.n_poses]
+
+            current_dict['original'] = poses[i: i + args.n_poses]
+            current_dict['latent_linear'] = encoded[i: i + args.n_poses].detach().clone().to('cpu').numpy()
+
+            input_pre_seq = encoded[i]
+            output_seq = encoded[i:i + args.n_poses]
+
+            input_seq = torch.unsqueeze(input_seq, 0)
+            input_seq = input_seq.transpose(0, 1)
+            use_drivitive = args.use_derivitive=='True'
+            if use_drivitive:
+                diff = [(input_seq[n, :] - input_seq[n - 1, :]) for n in range(1, input_seq.shape[0])]
+                diff.insert(0, torch.zeros_like(input_seq[0, :]))
+                input_seq = torch.cat((input_seq, torch.stack(diff)), dim=2)
+
+            # output_seq = torch.unsqueeze(output_seq, 0)
+            # output_seq = output_seq.transpose(0, 1)
+            output_seq = input_seq.clone()
+
+
+
+
+            reconstructed_rnn = torch.zeros(args.n_poses, output_seq.size(1), rnn.decoder.output_size) \
+                .to(output_seq.device)
+            # run words through encoder
+            encoder_outputs, encoder_hidden = rnn.encoder(input_seq, None)
+
+            if rnn.VAE:
+                decoder_hidden = encoder_hidden[:rnn.decoder.n_layers]  # use last hidden state from encoder
+                # [2, 128, 200]
+                # print("decoder_hidden!!! org", decoder_hidden.shape)
+                decoder_hidden = decoder_hidden.transpose(1, 0).contiguous()  # [128, 2, 200]
+                decoder_hidden = torch.reshape(decoder_hidden, (decoder_hidden.shape[0], -1))
+                mean = rnn.VAE_fc_mean(decoder_hidden)
+                logvar = rnn.VAE_fc_std(decoder_hidden)
+                z = rnn.reparameterize(mean, logvar, train=False)
+                z = rnn.VAE_fc_decoder(z)
+                decoder_hidden = z.reshape(decoder_hidden.shape[0],
+                                           rnn.decoder.n_layers, -1)
+                decoder_hidden = decoder_hidden.transpose(1, 0).contiguous()
+                # print("decoder_hidden!!! modified", decoder_hidden.shape)
+                decoder_first_hidden = decoder_hidden
+            else:
+                decoder_hidden = encoder_hidden[:rnn.decoder.n_layers]  # use last hidden state from encoder
+                # print("decoder_hidden!!! not VAE ", decoder_hidden.shape)
+
+            current_dict['latent_rnn'] = torch.squeeze(decoder_hidden.detach().clone(), 1).to('cpu').numpy()
+            result[result_index] = torch.squeeze(decoder_hidden.detach().clone(), 1).to('cpu').numpy()
+            result_index = result_index + 1
+
+        return result
+
+    def GPT_3_caller(self, str_input, pickle_file_loaded):
+
+        return 1
+
+        if pickle_file_loaded != None:
+            for i in range(len(pickle_file_loaded['sample_words_list'])):
+                if str_input == pickle_file_loaded['sample_words_list'][i]:
+                    print("Embedding found:", str_input)
+                    return pickle_file_loaded['GPT_3_Embedding_list'][i]
+
+        request = openai.Embedding.create(input=str_input, engine="text-similarity-ada-001")
+        embedding = request['data'][0]['embedding']
+        print("!!!!!!!!!!!!!!!!!!!Embedding requested:", str_input)
+        return embedding
+
diff --git a/scripts/data_loader/gesture_labels.txt b/scripts/data_loader/gesture_labels.txt
new file mode 100644
index 0000000..016b546
--- /dev/null
+++ b/scripts/data_loader/gesture_labels.txt
@@ -0,0 +1,411 @@
+payam,8720,1210,2930,neither,24.5149699
+payam,960,6830,7040,right,19.103924
+payam,2400,7930,7770,right,12.2262586
+payam,4850,2520,2830,neither,15.1091679
+payam,740,2820,2750,neither,18.6786855
+payam,1840,1090,7750,left,7.826561
+payam,7290,6890,6760,left,18.6590548
+payam,910,7710,7630,neither,15.9142669
+payam,110,3080,3020,right,19.3148321
+payam,4310,8430,6020,right,18.3946849
+payam,2370,6450,6640,neither,8.0961591
+payam,3760,3670,8090,left,35.6075189
+payam,7290,6890,6760,left,6.1512555
+payam,2330,1610,4290,neither,9.3917529
+payam,4960,1360,6940,neither,17.0703494
+payam,4880,2220,4050,neither,6.66984
+payam,4150,6320,5120,left,11.1385341
+payam,2380,2290,3830,neither,8.175581
+payam,6990,4350,7850,right,24.8897172
+payam,1260,1300,1150,left,8.7496713
+payam,4040,8150,270,neither,6.5148909
+payam,8210,4200,540,neither,9.3981568
+payam,1840,1090,7750,left,3.4069544
+payam,80,1770,5440,neither,6.8139126
+payam,4740,6380,6320,left,15.6245139
+payam,350,1710,820,neither,6.9307515
+payam,7980,380,440,right,6.9549693
+payam,4710,4520,6560,left,15.8368305
+payam,4150,1030,5450,right,5.8812649
+payam,4090,8210,1650,left,9.3473628
+payam,3160,3050,3300,right,15.4507672
+payam,3700,5340,4990,neither,19.4590834
+payam,5790,3250,4560,right,5.8536474
+payam,2820,2730,410,left,8.6303164
+payam,4180,6270,6780,neither,10.6426271
+payam,3450,3010,3200,neither,9.5873883
+payam,1480,2740,2400,neither,7.4411407
+payam,5200,140,2280,neither,10.3224914
+payam,7920,5360,5370,right,12.6515
+payam,8140,6670,6500,neither,8.3602584
+payam,1880,1810,5430,right,23.2110811
+payam,3190,1380,3600,left,13.4339459
+payam,50,4340,5020,neither,21.3263438
+payam,8350,8660,720,neither,5.8660808
+payam,3380,3280,1440,neither,12.1587357
+payam,1990,1450,3180,right,11.0167687
+payam,5220,3470,2030,right,17.4336008
+payam,3030,1430,7950,neither,5.1681943
+payam,2070,2690,1080,right,7.0990855
+payam,5270,760,2620,right,7.8710579
+payam,1500,5160,5060,right,13.7366622
+payam,3720,5320,980,right,3.9487614
+payam,6360,5830,6430,right,16.9997513
+payam,8100,2890,2200,neither,11.3478833
+payam,8190,8240,5770,left,9.2111841
+payam,6660,4820,3920,right,7.2671531
+payam,1380,1480,6320,left,8.1044273
+payam,2920,6550,6590,right,4.2769958
+payam,8030,5080,8270,left,8.6034509
+payam,7140,7510,4590,neither,16.6477046
+payam,2820,6050,8100,left,24.6404202
+payam,8760,7440,5770,right,13.7314397
+payam,1500,3220,670,left,11.3452669
+payam,7760,7550,5210,neither,6.2482816
+payam,1480,3220,1500,right,12.5323615
+payam,3370,5490,770,neither,7.1390339
+payam,7810,1860,900,right,10.9310926
+payam,1480,60,1000,left,22.48273
+payam,880,370,100,right,6.3977794
+payam,6530,6470,8270,left,23.1602921
+payam,7870,5570,5710,neither,10.2806256
+payam,6440,5560,1420,left,6.4094853
+payam,3080,3090,5170,left,21.2385869
+payam,2360,3630,3810,neither,8.7701795
+payam,490,1260,1220,neither,6.672139
+payam,2290,610,2460,left,6.049926
+payam,3380,3280,1440,neither,20.8177628
+payam,2660,6610,6450,left,42.8382073
+payam,5880,2790,2520,right,9.3886082
+payam,6520,5860,5480,neither,7.0841901
+payam,3640,2700,1130,left,32.19488
+payam,2800,6870,6710,neither,11.8798787
+payam,740,2820,2750,neither,9.5489998
+payam,7590,7770,6800,neither,19.0458094
+payam,4240,2560,270,right,12.5144577
+payam,4170,8630,3110,neither,9.0806503
+payam,4870,6970,6740,neither,16.2745591
+payam,6840,6060,6290,neither,6.9093279
+payam,90,3320,4740,neither,9.6742201
+payam,7140,7510,4590,left,22.7795439
+payam,5010,2290,7770,neither,19.3744291
+payam,5960,6240,8600,right,9.0793041
+payam,1250,6260,7880,right,4.93853
+payam,4270,1320,1350,right,17.2790887
+payam,4670,1250,3220,neither,10.618701
+payam,1480,2740,2400,neither,7.0960476
+payam,4270,1320,1350,right,4.4593416
+payam,7500,6670,450,neither,6.8258252
+payam,2330,3690,3800,neither,9.1139991
+payam,6890,3890,3700,neither,4.0416935
+payam,2160,8690,5990,neither,11.627078
+payam,5230,5350,5200,left,12.7237877
+payam,5410,5190,5000,neither,14.5101507
+payam,8310,8680,3050,left,7.9895167
+payam,1600,400,980,neither,10.7849002
+payam,170,2250,1110,right,8.0488411
+payam,540,2300,1470,left,15.8741313
+payam,6520,1530,2860,neither,7.0883511
+payam,5640,5810,4930,neither,7.4146026
+payam,3220,6620,5720,right,7.7339321
+test,30,3890,1900,right,22.9397707
+test,8530,4240,8100,neither,5.9101173
+test,7420,150,560,neither,137.2592382
+test,2130,5290,6180,neither,10.5838116
+test,1370,1510,1710,neither,6.7253406
+test,2750,1950,3380,right,17.9191102
+test,6460,2490,2850,neither,29.0007036
+test,3030,1430,7950,neither,3.9712144
+test,1390,7990,5990,right,42.9268688
+Payam,7010,7880,1480,neither,29.2939266
+Payam,3670,4380,1690,neither,13.5852305
+payam3,510,3840,2050,neither,19.60391
+payam3,3120,5240,400,neither,7.5961915
+payam3,6610,1980,5910,neither,32.5094664
+payam3,2790,6640,7410,neither,6.8419322
+payam3,2500,2680,520,neither,4.9135896
+payam3,3000,6310,6340,right,15.3105667
+payam3,3430,5560,80,neither,7.3489193
+payam3,5430,6460,570,neither,18.0880914
+payam3,5710,2830,2520,neither,11.3542066
+payam3,8740,1560,590,neither,9.5529691
+payam3,260,5430,4780,neither,8.8140994
+payam3,5380,5730,6680,neither,9.4738672
+payam3,990,190,3210,neither,6.6699878
+payam3,820,1910,4510,left,6.098936
+payam3,610,5890,7960,right,13.1935591
+payam3,4870,4910,6980,left,79.8618806
+payam3,120,1450,1240,left,23.4519851
+payam3,6940,7090,4180,left,7.9929938
+payam3,6360,5830,6430,neither,27.7130248
+payam,6980,8150,440,left,27.8841035
+payam,580,7490,7090,neither,10.9262192
+payam,2340,8460,6140,neither,26.7986099
+payam,5860,5670,4210,neither,18.7140217
+payam,1860,7700,7940,neither,14.8113515
+payam,8140,6110,4520,neither,24.3878234
+payam,2710,300,430,neither,21.5049204
+payam,3630,3590,2680,left,23.1192547
+payam,3740,820,3660,right,5.682812
+payam,1480,2740,2400,right,12.0348692
+payam,910,7710,7630,right,7.8845036
+payam,230,1540,4340,left,7.4580693
+payam,3170,1410,490,neither,10.9175666
+payam,500,2870,260,neither,12.8639616
+payam,7930,7780,5110,right,26.1968788
+payam,2300,1100,1090,right,8.9982824
+payam,2960,8730,4210,neither,15.1996149
+payam,4500,5370,2490,neither,20.4606212
+payam,2440,1140,6330,left,8.8519273
+payam,3930,1500,2990,right,12.3335319
+payam,4960,7920,3680,left,8.5003094
+payam,8600,6700,7470,right,8.8673901
+payam,5580,6610,1700,left,23.4709146
+payam,1320,3160,1380,right,19.3742774
+payam,8220,8660,1510,left,4.5353338
+payam,3120,6550,1760,neither,11.938804
+payam,3120,6550,1760,neither,24.8215028
+payam,3120,6550,1760,neither,32.6924222
+payam,3120,6550,1760,neither,199.0291749
+aaa,7680,780,3320,neither,13.9752481
+aaa,1170,1680,8400,right,72.1732308
+aaa,190,1570,590,neither,23.8951871
+aaa,890,190,8790,neither,15.8835947
+aaa,1110,850,1120,neither,12.2388968
+aaa,6990,70,840,neither,10.1972333
+aaa,1570,600,1040,neither,58.0329853
+payam,2610,3470,8790,neither,23.478667
+payam,2820,970,2610,neither,17.2883431
+payam,5130,3330,1970,neither,25.0759505
+payam,200,970,3200,neither,10.951438
+payam,6160,890,870,neither,13.3928642
+payam,4030,870,190,right,21.7972779
+payam,4640,3260,4480,neither,15.6459803
+payam,3750,3400,250,left,15.7365793
+payam,5820,5110,260,neither,11.9040374
+payam,4960,7920,2720,left,13.7575275
+payam,3340,5410,4670,neither,11.0850437
+payam,3220,5500,3450,neither,12.7476892
+payam,4830,3340,1750,neither,8.2270027
+payam,3280,5500,6100,neither,11.6609682
+payam,5570,3380,4830,neither,12.5343834
+payam,2170,3490,5410,left,20.2327963
+payam,1310,7950,5980,right,17.9011993
+payam,4830,8250,5060,right,12.6581358
+payam,2410,1730,1420,left,8.4628379
+payam,1610,140,1040,neither,39.4259191
+payam,2720,290,660,neither,9.004165
+payam,290,1770,1500,neither,8.7271764
+payam,2590,1750,2280,left,21.5393153
+payam,940,4110,5100,neither,14.6486691
+payam,3880,5100,8100,right,6.654937
+payam,2820,4740,3760,neither,8.9366303
+payam,2620,3800,3640,neither,32.7218192
+payam,490,3630,50,neither,15.6962632
+payam,6500,4740,3950,neither,10.4569726
+payam,3860,3960,3670,neither,5.6011954
+payam,8160,4670,3320,neither,9.9990773
+payam,5130,4670,3960,neither,10.0627506
+payam,1970,3670,6110,neither,8.9863573
+payam,5240,3800,4670,neither,35.1511656
+pp,6520,3000,3730,neither,13.2667115
+pp,3950,4670,8530,neither,10.3694821
+pp,3960,3670,1100,right,14.2283085
+pp,1080,2690,4240,left,8.0165702
+pp,4060,8590,7870,right,39.0669929
+pp,2730,860,5670,neither,21.0611187
+pp,1970,4410,2750,neither,23.8645963
+pp,2520,4410,7870,neither,7.0013341
+pp,2280,2790,2800,right,10.0719624
+pp,5730,1590,1630,neither,60.7203313
+pp,2170,2630,970,right,15.679544
+pp,7170,4640,1940,neither,12.8070448
+pp,1910,3790,3370,right,10.4345838
+pp,1940,5050,4170,neither,15.4618189
+pp,4240,6180,5690,neither,11.6433038
+pp,1360,4960,4100,right,22.1847562
+pp,2620,3610,3280,neither,16.611427
+payam,1140,1770,140,neither,32.3426861
+payam,5410,2110,3480,neither,9.2396016
+payam,6350,4540,2160,neither,15.8730432
+payam,1090,3280,5400,right,7.9992393
+payam,6870,7500,6480,left,16.2428709
+payam,4890,7850,2720,left,11.0865865
+payam,6340,6250,450,left,7.1841906
+payam,2410,1240,1360,right,3.8704583
+payam,4010,8090,2620,left,11.8036291
+payam,1940,4910,4050,right,10.9399087
+payam,8580,6390,5740,neither,19.4242683
+payam,360,6430,6520,neither,39.232415
+payam,4330,5830,6420,neither,14.8946759
+payam,7070,6430,280,neither,9.1711564
+payam,440,6390,5570,neither,16.4782124
+payam,5830,6650,3080,neither,27.4215719
+payam,6620,5790,3630,neither,7.9312582
+payam,3520,6680,5860,neither,15.903332
+payam,6680,6600,1570,neither,7.0110227
+payam,1050,5650,5570,neither,9.8728923
+payam,4540,5740,6430,right,7.6298997
+payam,2560,1730,1030,left,64.8227024
+payam,1910,6760,7300,right,5.3704937
+payam,5730,3020,3080,right,14.8289519
+payam,4440,1750,1640,neither,17.3421493
+payam,3160,910,140,neither,7.7735092
+payam,5280,80,2260,right,10.4654106
+p,8630,3910,1190,neither,12.2734672
+p,4610,8400,6030,neither,25.0126291
+p,4250,5200,8530,right,11.2253927
+p,370,660,6480,neither,8.051555
+p,5420,150,180,right,7.0392668
+p,3360,3600,560,right,12.5866966
+p,3900,3980,3760,right,60.7114951
+p,5010,8340,6970,neither,25.5606582
+p,6210,8650,2790,neither,14.1406681
+p,640,8210,5010,neither,11.7996506
+p,3200,8170,8030,neither,88.1912972
+p,7970,8110,7150,neither,9.3730532
+p,7980,4090,8290,right,14.0371127
+p,1070,430,410,neither,9.7275574
+p,310,410,4240,left,5.8301648
+p,7030,2460,2840,right,7.3928905
+p,3830,5260,3130,neither,7.6125024
+p,3850,4450,7780,neither,15.566987
+p,3830,2120,6260,neither,15.1221466
+p,5340,4510,1340,neither,10.4645626
+p,3830,3710,8580,neither,10.3860476
+p,7090,2180,3720,neither,13.3298749
+p,3410,2120,6400,neither,6.5504196
+p,1570,1990,3880,left,16.0690818
+p,8280,7720,7760,left,19.2019649
+p,180,4850,980,right,12.3407615
+payam,4460,2880,5160,neither,31.7953241
+payam,4990,4230,1360,left,26.7975901
+payam,3690,2640,2750,neither,13.5067165
+payam,2820,2520,5540,neither,14.3831229
+payam,640,2830,2670,neither,22.702336
+payam,2640,2520,1790,right,9.7770384
+payam,2220,3330,4770,neither,17.5712949
+payam,4000,2130,4350,neither,8.9440079
+payam,2470,5370,4710,neither,25.1191583
+payam,60,4790,3780,neither,6.8597862
+payam,5290,4760,4360,neither,39.3547437
+payam,2680,5470,4730,right,9.0767636
+payam,460,1700,4490,left,9.5737932
+payam,640,4640,3240,left,9.0991829
+payam,5360,8340,7920,neither,43.4682622
+payam,8370,8320,5650,left,13.9077435
+payam,8800,6350,2520,neither,23.3640544
+pp,1270,4440,2720,neither,15.9654004
+pp,6050,5870,8630,neither,64.0813694
+pp,4960,4100,1380,left,20.2915712
+pp,2410,2030,3340,right,5.8915843
+pp,7520,6470,3160,left,4.4095175
+pp,2490,7850,5920,right,19.7446139
+pp,7870,8190,6080,left,12.2258913
+payam,6910,8130,5200,left,22.1673536
+payam,1270,1310,8550,neither,4.9356615
+payam,1200,3010,8090,neither,6.2535121
+payam,1640,1200,3010,neither,7.2112379
+payam,2160,3180,1450,neither,12.2517686
+payam,2290,2920,1450,neither,11.0051224
+payam,4620,2970,1270,neither,6.0543814
+payam,3180,1510,360,neither,7.1203205
+payam,1110,3010,1200,neither,4.9222736
+payam,500,1240,2970,neither,244.7143959
+payam,1370,2970,1540,neither,69.5212346
+payam,3010,3200,5450,neither,12.7399181
+payam,3120,1270,1240,right,7.1447759
+payam,4860,8750,5670,neither,19.6800883
+payam,190,8410,8790,right,4.0843236
+payam,4620,7980,4870,neither,11.6172416
+payam,4870,5080,1040,neither,9.651124
+payam,590,4910,4880,neither,17.2420091
+payam,340,4880,5080,neither,11.3620724
+payam,8740,4050,4020,right,71.2654567
+payam,2310,1960,6000,neither,21.7536178
+payam,6620,5740,5500,neither,6.9649316
+payam,5670,5650,5300,right,7.7041501
+payam,3040,7940,7700,neither,25.42975
+payam,7710,7780,1620,left,195.092137
+payam,7430,7320,5660,right,13.8465693
+payam,7980,2240,2230,right,13.2360946
+payam,6790,7300,3100,neither,15.0350203
+payam,2650,7200,7300,right,6.4745868
+payam,1430,1440,2920,neither,7.7909788
+payam,3060,1290,2590,left,56.5645663
+payam,6500,790,2720,neither,12.4153127
+payam,1750,1740,7740,neither,45.0387307
+payam,3860,290,1750,right,127.7812023
+payam,2900,3120,3160,left,8.5753824
+payam,560,2300,2550,right,18.1986405
+payam,1210,6030,8530,neither,12.6016999
+payam,5130,8400,4060,right,13.3131464
+payam,1060,320,250,right,11.4700255
+payam,3250,4560,7510,left,9.6844356
+payam,2140,1010,2570,neither,19.8406119
+payam,1000,400,1450,neither,20.5022202
+payam,930,900,5540,left,9.8994706
+payam,3760,3520,1860,right,47.1336049
+payam,6140,1230,1260,left,108.9202734
+payam,590,430,310,neither,11.3733376
+payam,7880,2690,2680,neither,5.1495561
+payam,3170,2680,310,neither,8.5025821
+payam,3050,1080,410,neither,8.527219
+payam,2770,1080,7490,neither,402.8720653
+payam,3190,410,2680,neither,10.4691005
+payam,2770,410,3720,left,4.3418474
+payam,1410,2460,340,neither,12.1507813
+payam,3150,2440,2840,neither,11.3710139
+payam,2820,800,2310,neither,30.0199888
+payam,2460,710,160,neither,9.4166814
+payam,2440,30,6480,neither,7.3646978
+payam,7870,1140,340,neither,14.3368254
+payam,2930,1720,2590,neither,18.4583712
+payam,240,2450,5190,left,8.7122048
+payam,3890,2120,6000,right,17.7283225
+payam,6190,4120,6640,left,12.888971
+payam,5240,3650,5320,right,4.7470391
+payam,4330,5950,6030,left,9.3768817
+payam,1110,590,430,left,11.9393439
+payam,3000,4830,8350,neither,17.2718661
+payam,1790,2730,2750,neither,9.0715062
+payam,2820,2760,5640,neither,136.1871848
+payam,2790,2830,0,left,7.9951019
+payam,6890,3350,5370,right,10.1746638
+payam,4100,460,850,neither,15.272502
+payam,2880,190,2630,neither,11.2373856
+payam,7290,1590,460,right,328.9525274
+payam,3270,2010,590,right,42.705704
+payam,500,6020,8370,neither,10.2211368
+payam,6160,8320,2700,neither,5.8553287
+payam,2910,8510,6260,neither,20.3620241
+payam,8660,5940,6070,left,10.4425546
+payam,6670,4170,4960,right,28.4874515
+payam,1350,2030,5410,neither,8.5564246
+payam,2160,2110,2680,left,5.1389665
+payam,7520,6530,4530,neither,13.91322
+payam,7230,6670,2250,neither,8.7192474
+payam,7500,7230,8500,neither,12.1839644
+payam,6530,7500,7660,neither,13.3554725
+payam,6670,7520,7950,left,7.8109235
+payam,4440,6070,1490,neither,22.8200555
+payam,6290,4350,530,left,30.2692709
+payam,7870,8190,6080,left,18.4469568
+payam,8520,3180,2920,neither,15.4442971
+payam,1450,3180,1960,left,14.0648741
+payam,1770,8490,4180,neither,15.9577787
+payam,5060,8790,3120,neither,19.0112589
+payam,4310,8230,3100,neither,18.039143
+payam,8720,7890,5100,right,27.7411957
+payam,2620,8740,8270,neither,10.0985898
+payam,5080,4870,8800,neither,285.7399943
+payam,390,4910,4880,neither,31.1095841
+payam,4880,8270,2720,neither,6.3076338
+payam,3030,4880,8740,neither,17.0361449
+payam,4910,7980,2910,neither,8.4894469
+payam,4050,7980,1710,neither,6.3573893
+payam,4910,4880,5450,neither,4.4998203
+payam,1750,4050,8740,right,96.8739301
+payam,6650,6620,5540,neither,11.0029129
+payam,5740,5830,3310,neither,12.8770997
+payam,6390,6560,1970,neither,40.2096949
diff --git a/scripts/data_loader/lmdb_data_loader.py b/scripts/data_loader/lmdb_data_loader.py
new file mode 100644
index 0000000..ed3e883
--- /dev/null
+++ b/scripts/data_loader/lmdb_data_loader.py
@@ -0,0 +1,746 @@
+import logging
+import os
+import pickle
+
+import numpy as np
+import lmdb as lmdb
+import torch
+from torch.nn.utils.rnn import pad_sequence
+
+from torch.utils.data import Dataset
+from torch.utils.data.dataloader import default_collate
+
+from data_loader.data_preprocessor import DataPreprocessor
+import pyarrow
+import random
+from tslearn.clustering import TimeSeriesKMeans
+import utils
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+def word_seq_collate_fn(data):
+    """ collate function for loading word sequences in variable lengths """
+    # sort a list by sequence length (descending order) to use pack_padded_sequence
+    data.sort(key=lambda x: len(x[0]), reverse=True)
+
+
+    # separate source and target sequences
+    # Todo: fix this using args
+    sentence_leve = True
+    if not sentence_leve:
+        word_seq, poses_seq, audio, aux_info, = zip(*data)
+    else:
+        word_seq, poses_seq, audio, aux_info, sentence_leve_latents, cluster_portion, GPT3_Embedding = zip(*data)
+
+
+
+
+    # merge sequences
+    words_lengths = torch.LongTensor([len(x) for x in word_seq])
+    word_seq = pad_sequence(word_seq, batch_first=True).long()
+
+    poses_seq = default_collate(poses_seq)
+    audio = default_collate(audio)
+    if sentence_leve:
+        sentence_leve_latents = default_collate(sentence_leve_latents)
+        cluster_portion = default_collate(cluster_portion)
+        GPT3_Embedding = default_collate(GPT3_Embedding)
+
+    # audio = default_collate(audio)
+    aux_info = {key: default_collate([d[key] for d in aux_info]) for key in aux_info[0]}
+
+    if sentence_leve:
+        return word_seq, words_lengths, poses_seq, audio, aux_info, sentence_leve_latents, cluster_portion, GPT3_Embedding
+    else:
+        return word_seq, words_lengths, poses_seq, audio, aux_info
+
+
+class TrinityDataset(Dataset):
+    def __init__(self, args, lmdb_dir, n_poses, subdivision_stride, pose_resampling_fps, data_mean, data_std):
+
+        self.lmdb_dir = lmdb_dir
+        self.n_poses = n_poses
+        self.subdivision_stride = subdivision_stride
+        self.skeleton_resampling_fps = pose_resampling_fps
+        self.lang_model = None
+        self.data_mean = np.array(data_mean).squeeze()
+        self.data_std = np.array(data_std).squeeze()
+
+        logging.info("Reading data '{}'...".format(lmdb_dir))
+        preloaded_dir = lmdb_dir + '_cache'
+        if not os.path.exists(preloaded_dir):
+            data_sampler = DataPreprocessor(args, lmdb_dir, preloaded_dir, n_poses,
+                                            subdivision_stride, pose_resampling_fps, sentence_level=False)
+            data_sampler.run()
+        else:
+            logging.info('Found pre-loaded samples from {}'.format(preloaded_dir))
+
+        # init lmdb
+        self.lmdb_env = lmdb.open(preloaded_dir, readonly=True, lock=False)
+        with self.lmdb_env.begin() as txn:
+            self.n_samples = txn.stat()['entries']
+
+    def __len__(self):
+        return self.n_samples
+
+    def __getitem__(self, idx):
+        with self.lmdb_env.begin(write=False) as txn:
+            key = '{:010}'.format(idx).encode('ascii')
+            sample = txn.get(key)
+
+            sample = pyarrow.deserialize(sample)
+            word_seq, pose_seq, audio, aux_info = sample
+
+        def words_to_tensor(lang, words, end_time=None):
+            indexes = [lang.SOS_token]
+            for word in words:
+                if end_time is not None and word[1] > end_time:
+                    break
+                indexes.append(lang.get_word_index(word[0]))
+            indexes.append(lang.EOS_token)
+            return torch.Tensor(indexes).long()
+
+        # normalize
+        std = np.clip(self.data_std, a_min=0.01, a_max=None)
+        pose_seq = (pose_seq - self.data_mean) / std
+
+        # to tensors
+        word_seq_tensor = words_to_tensor(self.lang_model, word_seq, aux_info['end_time'])
+        pose_seq = torch.from_numpy(pose_seq).reshape((pose_seq.shape[0], -1)).float()
+        # audio = torch.from_numpy(audio).float()
+
+        return word_seq_tensor, pose_seq, audio, aux_info
+
+    def set_lang_model(self, lang_model):
+        self.lang_model = lang_model
+
+class TrinityDataset_DAE(Dataset):
+    def __init__(self, args, lmdb_dir, n_poses, subdivision_stride, pose_resampling_fps, data_mean, data_std):
+
+        self.lmdb_dir = lmdb_dir
+        self.n_poses = n_poses
+        self.subdivision_stride = subdivision_stride
+        self.skeleton_resampling_fps = pose_resampling_fps
+        self.lang_model = None
+        self.data_mean = np.array(data_mean).squeeze()
+        self.data_std = np.array(data_std).squeeze()
+
+        logging.info("Reading data '{}'...".format(lmdb_dir))
+        preloaded_dir = lmdb_dir + '_cache'
+        if not os.path.exists(preloaded_dir):
+
+            data_sampler = DataPreprocessor(args, lmdb_dir, preloaded_dir, n_poses,
+                                            subdivision_stride, pose_resampling_fps)
+            data_sampler.run()
+        else:
+            logging.info('Found pre-loaded samples from {}'.format(preloaded_dir))
+
+        # init lmdb
+        self.lmdb_env = lmdb.open(preloaded_dir, readonly=True, lock=False)
+        with self.lmdb_env.begin() as txn:
+            self.n_samples = txn.stat()['entries']
+
+
+        # Initiate all poses
+        # self.all_poses = []
+        # self.create_all_poses()
+        # print("data init finished!")
+
+
+    def __len__(self):
+        # return 500
+        return (self.n_samples * self.n_poses) //5
+        return self.n_samples
+
+    def create_all_poses(self):
+        with self.lmdb_env.begin(write=False) as txn:
+            for i in range(self.n_samples):
+                key = '{:010}'.format(i).encode('ascii')
+                sample = txn.get(key)
+
+                sample = pyarrow.deserialize(sample)
+                word_seq, pose_seq, audio, aux_info = sample
+
+                # normalize
+                std = np.clip(self.data_std, a_min=0.01, a_max=None)
+                pose_seq = (pose_seq - self.data_mean) / std
+
+                for j in range(0, len(pose_seq)):
+                    original = pose_seq[j,:]
+                    var_coef = 1 #std
+                    sigma = 1
+                    # noisy = self.add_noise(original, 0.0, std)
+                    noisy = original
+                    self.all_poses.append({'original': original, "noisy": noisy})
+
+
+    def get_item_Memory_Efficient(self, idx):
+        idx_lmdb = idx//self.n_poses
+
+        with self.lmdb_env.begin(write=False) as txn:
+            key = '{:010}'.format(idx_lmdb).encode('ascii')
+            sample = txn.get(key)
+
+            sample = pyarrow.deserialize(sample)
+            word_seq, pose_seq, audio, aux_info = sample
+
+
+            # normalize
+            std = np.clip(self.data_std, a_min=0.01, a_max=None)
+            pose_seq = (pose_seq - self.data_mean) / std
+
+            original = pose_seq[idx % self.n_poses, :]
+            noisy = original
+
+            original = torch.from_numpy(original).reshape((original.shape[0], -1)).float()
+            noisy = torch.from_numpy(noisy).reshape((noisy.shape[0], -1)).float()
+
+            return noisy, original
+
+    def add_noise(self, x, variance_multiplier, sigma):
+        """
+               Add Gaussian noise to the data
+               Args:
+                   x                   - input vector
+                   variance_multiplier - coefficient to multiple variance of the noise on
+                   sigma               - variance of the dataset
+               Returns:
+                   x - output vector, noisy data
+        """
+        eps = 1e-15
+        noise = np.random.normal(0.0, np.multiply(sigma, variance_multiplier) + eps, x.shape)
+        x = x + noise
+        return x
+    def add_noise2(self, x, prob):
+        """
+               Add Gaussian noise to the data
+               Args:
+                   x                   - input vector
+                   variance_multiplier - coefficient to multiple variance of the noise on
+                   sigma               - variance of the dataset
+               Returns:
+                   x - output vector, noisy data
+        """
+
+        for i in range(len(x)):
+            rnd = random.random()
+            if (rnd < prob):
+                x[i] = 0
+        return x
+
+    def __getitem__(self, idx):
+
+        # Todo: I think this way is more efficient than the prev. approach
+        # I need to double check and make sure everything works as before.
+        return self.get_item_Memory_Efficient(idx)
+        # original, noisy = self.get_item_Memory_Efficient(idx)
+        # return noisy, original
+
+
+        original = self.all_poses[idx]['original']
+        noisy = self.all_poses[idx]['noisy']
+
+        original = torch.from_numpy(original).reshape((original.shape[0], -1)).float()
+        noisy = torch.from_numpy(noisy).reshape((noisy.shape[0], -1)).float()
+
+        return noisy, original
+
+    def set_lang_model(self, lang_model):
+        self.lang_model = lang_model
+
+
+class TrinityDataset_DAEed_Autoencoder(Dataset):
+    def __init__(self, args, lmdb_dir, n_poses, subdivision_stride, pose_resampling_fps, data_mean, data_std):
+
+        self.lmdb_dir = lmdb_dir
+        self.n_poses = n_poses
+        self.subdivision_stride = subdivision_stride
+        self.skeleton_resampling_fps = pose_resampling_fps
+        self.lang_model = None
+        self.data_mean = np.array(data_mean).squeeze()
+        self.data_std = np.array(data_std).squeeze()
+
+        # We will change it to true once we call the creat_similarity_dataset
+        self.pairwise_enabeled = False
+        self.use_drivitive = args.use_derivitive == 'True'
+
+        logging.info("Reading data '{}'...".format(lmdb_dir))
+        preloaded_dir = lmdb_dir + '_cache'
+        if not os.path.exists(preloaded_dir):
+            data_sampler = DataPreprocessor(args, lmdb_dir, preloaded_dir, n_poses,
+                                            subdivision_stride, pose_resampling_fps)
+            data_sampler.run()
+        else:
+            logging.info('Found pre-loaded samples from {}'.format(preloaded_dir))
+
+        # init lmdb
+        self.lmdb_env = lmdb.open(preloaded_dir, readonly=True, lock=False)
+        with self.lmdb_env.begin() as txn:
+            self.n_samples = txn.stat()['entries']
+
+    #     Todo: we need to initiate pre-trained representation learning model
+        checkpoint_path = args.rep_learning_checkpoint
+        self.rep_learning_dim = args.rep_learning_dim
+        rep_learning_args, rep_model, rep_loss_fn, rep_lang_model, rep_out_dim = utils.train_utils.load_checkpoint_and_model(
+            checkpoint_path, device, 'DAE')
+        self.rep_model = rep_model.to('cpu')
+        self.rep_model.train(False)
+
+    def __len__(self):
+        return self.n_samples//10
+
+    def __getitem__(self, idx):
+        with self.lmdb_env.begin(write=False) as txn:
+            key = '{:010}'.format(idx).encode('ascii')
+            sample = txn.get(key)
+
+            sample = pyarrow.deserialize(sample)
+            word_seq, pose_seq, audio, aux_info = sample
+
+        # normalize
+        std = np.clip(self.data_std, a_min=0.01, a_max=None)
+        pose_seq = (pose_seq - self.data_mean) / std
+
+
+        # Todo: Here we should apply rep_learning
+        target = torch.from_numpy(pose_seq)
+        # target = torch.unsqueeze(target, 2)
+        target = target.float()
+        # target = target.to(device)
+        with torch.no_grad():
+            if self.rep_model.encoder == None: #for ablation study
+                encoded_poses = target
+            else:
+                encoded_poses = self.rep_model.encoder(target)
+
+        # encoded_poses = torch.squeeze(encoded_poses, 2)
+        # encoded_poses = encoded_poses.to('cpu')
+        # reconstructed = encoded_poses.detach().numpy()
+
+        # to tensors
+        # word_seq_tensor = words_to_tensor(self.lang_model, word_seq, aux_info['end_time'])
+        # pose_seq = torch.from_numpy(pose_seq).reshape((pose_seq.shape[0], -1)).float()
+            encoded_poses = encoded_poses.reshape((encoded_poses.shape[0], -1)).float()
+        # audio = torch.from_numpy(audio).float()
+
+
+        if self.use_drivitive:
+            diff =  [(encoded_poses[n, :] - encoded_poses[n - 1, :]) for n in range(1, encoded_poses.shape[0])]
+            diff.insert(0,torch.zeros_like(encoded_poses[0, :]))
+            encoded_poses = torch.hstack((encoded_poses, torch.stack(diff)))
+
+        # return word_seq_tensor, pose_seq, audio, aux_info
+        return encoded_poses, encoded_poses
+
+
+    def creat_similarity_dataset(self, pickle_file, labelstxt_file):
+        # Todo: 1. Thos function gets the pickle file that I made in the clustering.py(or flowgmm) process as well
+        # Todo: as the labels text file that I annotated in the Unity application.
+        # Todo: 2. Then I will creat those pairs of similarity and dissimilarity
+        # Todo: 3. Finally, we store the pairs into the class.
+        # Todo: We will use pairwise label and an extra loss in backpropagation process later.
+
+        # 1. call preprocess load
+        self.data_rnn, self.labels, self.pairwise_labels, self.data_original = self.load_gesture_data(pickle_file, labelstxt_file)
+
+        # normalize
+        std = np.clip(self.data_std, a_min=0.01, a_max=None)
+        self.data_original = ( self.data_original - self.data_mean) / std
+
+        target = torch.from_numpy(self.data_original)
+
+        target = target.float()
+        # target = target.to(device)
+        with torch.no_grad():
+            self.encoded_labeled_poses = self.rep_model.encoder(target)
+            if self.use_drivitive:
+                diff = [(self.encoded_labeled_poses[n, :] - self.encoded_labeled_poses[n - 1, :])
+                        for n in range(1, self.encoded_labeled_poses.shape[0])]
+                diff.insert(0, torch.zeros_like(self.encoded_labeled_poses[0, :]))
+                self.encoded_labeled_poses = torch.cat((self.encoded_labeled_poses, torch.stack(diff)), dim=2)
+        self.pairwise_enabeled = True
+        pass
+
+
+    def get_labeled_(self, count):
+        stack_pairs1 = torch.zeros(count, self.encoded_labeled_poses.shape[1], self.encoded_labeled_poses.shape[2])
+        stack_pairs2 = torch.zeros(count, self.encoded_labeled_poses.shape[1], self.encoded_labeled_poses.shape[2])
+        stack_labels = torch.zeros(count)
+        rnds = random.sample(range(1, len(self.pairwise_labels)), 3)
+        k=0
+        for rnd in rnds:
+            current_pair = self.pairwise_labels[rnd]
+            s1_ = self.encoded_labeled_poses[current_pair[0]]
+            s2_ = self.encoded_labeled_poses[current_pair[1]]
+            ss_label = current_pair[2]
+            stack_pairs1[k, :, :] = s1_
+            stack_pairs2[k, :, :] = s2_
+            stack_labels[k] = (ss_label)
+            k = k+1
+
+
+        return stack_pairs1, stack_pairs2, stack_labels
+
+    def set_lang_model(self, lang_model):
+        self.lang_model = lang_model
+
+    def load_gesture_data(self, pre_processed_pickle_adress, labelstxt_file):
+
+        loaded = pickle.load(open(pre_processed_pickle_adress, 'rb'))
+        liaded_len = len(loaded)
+        loaded = np.hstack(loaded)
+        print('len loaded', len(loaded))
+        print("Loaded successfully")
+
+        data_latent_rnn = []
+        data_latent_linear = np.zeros([len(loaded), loaded[0]['latent_linear'].shape[0],
+                                       loaded[0]['latent_linear'].shape[1]])
+        data_latent_linear_listwise = []
+        data_original = []
+        count = len(loaded)
+        # count = 4000
+        for i in range(count):
+
+            # 1
+            current_latent_linear = loaded[i]['latent_linear']
+            current_latent_rnn = loaded[i]['latent_rnn']
+            current_original = loaded[i]['original']
+
+            if len(current_original) != len(loaded[0]['original']):
+                continue
+
+            # 2
+            # current_latent_linear = np.hstack(current_latent_linear)
+            current_latent_rnn = np.hstack(current_latent_rnn)
+            # current_original = np.hstack(current_original)
+
+            # 3
+            data_latent_linear[i] = (current_latent_linear)
+            data_latent_linear_listwise.append(current_latent_linear)
+            data_latent_rnn.append(current_latent_rnn)
+            data_original.append(current_original)
+
+        # Should be constructed here since it is not obvious how many we will have at the end.
+        data_latent_linear = np.zeros([len(data_latent_linear_listwise), loaded[0]['latent_linear'].shape[0],
+                                       loaded[0]['latent_linear'].shape[1]])
+        for i in range(len(data_latent_linear_listwise)):
+            data_latent_linear[i] = data_latent_linear_listwise[i]
+
+        data_latent_rnn_ndarray = np.array(data_latent_rnn)
+
+        first_order_labels = np.ones(len(data_latent_rnn_ndarray)) * -1
+
+        labels_list = []
+        label_file = open(labelstxt_file, 'r')
+
+        for line in label_file:
+            str = line.split(',')
+            lbl = (str[4])
+            left = int(str[1])
+            middle = int(str[2])
+            right = int(str[3])
+            chance = random.random()
+            # if chance > 0.1:
+            #     continue
+            if (lbl == 'neither'):
+                # continue
+                labels_list.append([right, middle, 0])
+                labels_list.append([left, middle, 0])
+                first_order_labels[right] = 1
+                first_order_labels[left] = 1
+                first_order_labels[middle] = 1
+            if (lbl == 'right'):
+                labels_list.append([right, middle, 1])
+                first_order_labels[right] = 1
+                # first_order_labels[left] = 1
+                first_order_labels[middle] = 1
+            if (lbl == 'left'):
+                labels_list.append([left, middle, 1])
+                # first_order_labels[right] = 1
+                first_order_labels[left] = 1
+                first_order_labels[middle] = 1
+
+        #     Todo: read from file
+
+        return data_latent_rnn_ndarray[:, 0:200], first_order_labels, (labels_list), data_original
+
+
+class TrinityDataset_with_cluster(Dataset):
+    def __init__(self, args, lmdb_dir, n_poses, subdivision_stride, pose_resampling_fps, data_mean, data_std):
+
+        self.lmdb_dir = lmdb_dir
+        self.n_poses = n_poses
+        self.subdivision_stride = subdivision_stride
+        self.skeleton_resampling_fps = pose_resampling_fps
+        self.lang_model = None
+        self.data_mean = np.array(data_mean).squeeze()
+        self.data_std = np.array(data_std).squeeze()
+        self.use_drivitive = args.use_derivitive == 'True'
+        self.kmeanmodel = pickle.load(open('../output/clustering_results/kmeans_model.pk','rb'))
+
+        if args.sentence_level == 'True':
+            self.sentence_level = True
+        else:
+            self.sentence_level = False
+
+        logging.info("Reading data '{}'...".format(lmdb_dir))
+        if self.sentence_level:
+            preloaded_dir = lmdb_dir + '_sentence_level' + '_cache'
+        else:
+            preloaded_dir = lmdb_dir + '_cache'
+
+        if not os.path.exists(preloaded_dir):
+            data_sampler = DataPreprocessor(lmdb_dir, preloaded_dir, n_poses,
+                                            subdivision_stride, pose_resampling_fps)
+            data_sampler.run()
+        else:
+            logging.info('Found pre-loaded samples from {}'.format(preloaded_dir))
+
+        # init lmdb
+        self.lmdb_env = lmdb.open(preloaded_dir, readonly=True, lock=False)
+        with self.lmdb_env.begin() as txn:
+            self.n_samples = txn.stat()['entries']
+
+        # Representation model
+        checkpoint_path = args.rep_learning_checkpoint
+        self.rep_learning_dim = args.rep_learning_dim
+        rep_learning_args, rep_model, rep_loss_fn, rep_lang_model, rep_out_dim = utils.train_utils.load_checkpoint_and_model(
+            checkpoint_path, device, 'DAE')
+        self.rep_model = rep_model.to('cpu')
+        self.rep_model.train(False)
+        #   RNN autoencoder
+        checkpoint_path = args.autoencoder_checkpoint
+
+        # RNNAutoencoder_args, RNNAutoencoder_model, RNNAutoencoder_fn,\
+        # RNNAutoencoder_model, RNNAutoencoder_dim = utils.train_utils.load_checkpoint_and_model(
+        #     checkpoint_path, device)
+
+        args, rnn, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+            checkpoint_path, device, 'autoencoder')
+        self.RNNAutoencoder_model = rnn.to('cpu')
+        self.RNNAutoencoder_model.train(False)
+
+    def __len__(self):
+        return self.n_samples
+
+    def __getitem__(self, idx):
+        with self.lmdb_env.begin(write=False) as txn:
+            key = '{:010}'.format(idx).encode('ascii')
+            sample = txn.get(key)
+
+            sample = pyarrow.deserialize(sample)
+            word_seq, pose_seq, audio, aux_info, portion = sample
+
+        def words_to_tensor(lang, words, end_time=None):
+            indexes = [lang.SOS_token]
+            for word in words:
+                if end_time is not None and word[1] > end_time:
+                    break
+                indexes.append(lang.get_word_index(word[0]))
+            indexes.append(lang.EOS_token)
+            return torch.Tensor(indexes).long()
+
+        # normalize
+        std = np.clip(self.data_std, a_min=0.01, a_max=None)
+        pose_seq = (pose_seq - self.data_mean) / std
+
+        # Todo: Here we should apply rep_learning
+        target = torch.from_numpy(pose_seq)
+        target = target.float()
+        with torch.no_grad():
+            encoded_poses = self.rep_model.encoder(target)
+
+        encoded_poses = encoded_poses.reshape((encoded_poses.shape[0], -1)).float()
+
+        if self.use_drivitive:
+            diff = [(encoded_poses[n, :] - encoded_poses[n - 1, :]) for n in range(1, encoded_poses.shape[0])]
+            diff.insert(0, torch.zeros_like(encoded_poses[0, :]))
+            encoded_poses = torch.hstack((encoded_poses, torch.stack(diff)))
+        with torch.no_grad():
+            out_pose, latent, mue, logvar = self.RNNAutoencoder_model(encoded_poses.unsqueeze(0), encoded_poses.unsqueeze(0))
+            latent = latent[:self.RNNAutoencoder_model.decoder.n_layers]
+            latent = latent.squeeze(1)
+            latent = latent.reshape((1, -1)).float()
+            # latent = latent.squeeze(0)
+            cluster_id = self.kmeanmodel.predict(latent.cpu().detach().numpy())
+            cluster_id = torch.Tensor(cluster_id).long()
+        # to tensors
+        word_seq_tensor = words_to_tensor(self.lang_model, word_seq, aux_info['end_time'])
+        pose_seq = torch.from_numpy(pose_seq).reshape((pose_seq.shape[0], -1)).float()
+
+        # audio = torch.from_numpy(audio).float()
+
+        # return cluster_id
+        return word_seq_tensor, encoded_poses, audio, aux_info, cluster_id
+
+    def set_lang_model(self, lang_model):
+        self.lang_model = lang_model
+
+
+class TrinityDataset_sentencelevel(Dataset):
+    def __init__(self, args, lmdb_dir, n_poses, subdivision_stride, pose_resampling_fps, data_mean, data_std):
+
+        self.lmdb_dir = lmdb_dir
+        self.n_poses = n_poses
+        self.subdivision_stride = subdivision_stride
+        self.subdivision_stride_sentence = args.subdivision_stride_sentence
+        self.args = args
+        self.skeleton_resampling_fps = pose_resampling_fps
+        self.lang_model = None
+        self.data_mean = np.array(data_mean).squeeze()
+        self.data_std = np.array(data_std).squeeze()
+        self.use_drivitive = args.use_derivitive == 'True'
+
+        # Todo: fix adress -- Fixed
+        test = os.path.dirname(args.autoencoder_checkpoint)
+        self.kmeanmodel = pickle.load(open(os.path.dirname(args.autoencoder_checkpoint)
+                                           + '/clusters/kmeans_model.pk', 'rb'))
+        # self.clustering_transforms = pickle.load(open(os.path.dirname(args.autoencoder_checkpoint)
+        #                                               + '/clusters/transforms.pkl', 'rb'))
+
+        if args.sentence_level == 'True':
+            self.sentence_level = True
+        else:
+            self.sentence_level = False
+
+        logging.info("Reading data '{}'...".format(lmdb_dir))
+
+        if self.sentence_level:
+            # preloaded_dir = lmdb_dir + '_sentence_level' + '_cache'
+            preloaded_dir = args.model_save_path + 'lmdb/' + os.path.basename(
+                (lmdb_dir)) + '_sentence_level' + '_cache'
+        else:
+            # preloaded_dir = lmdb_dir + '_cache'
+            preloaded_dir = args.model_save_path + 'lmdb/' + os.path.basename(
+                (lmdb_dir)) + '_cache'
+
+
+        if not os.path.exists(args.model_save_path + 'lmdb'):
+            os.mkdir(args.model_save_path + 'lmdb')
+        if not os.path.exists(preloaded_dir):
+            data_sampler = DataPreprocessor(args, lmdb_dir, preloaded_dir, n_poses,
+                                            self.subdivision_stride_sentence, pose_resampling_fps,
+                                            sentence_level=self.sentence_level)
+            data_sampler.run()
+        else:
+            # Todo: Remove later
+
+            # data_sampler = DataPreprocessor(args, lmdb_dir, preloaded_dir, n_poses,
+            #                                 subdivision_stride, pose_resampling_fps,
+            #                                 sentence_level=self.sentence_level)
+            # data_sampler.run()
+            logging.info('Found pre-loaded samples from {}'.format(preloaded_dir))
+
+
+
+        # init lmdb
+        self.lmdb_env = lmdb.open(preloaded_dir, readonly=True, lock=False)
+        with self.lmdb_env.begin() as txn:
+            self.n_samples = txn.stat()['entries']
+
+        # Representation model
+        checkpoint_path = args.rep_learning_checkpoint
+        self.rep_learning_dim = args.rep_learning_dim
+        rep_learning_args, rep_model, rep_loss_fn, rep_lang_model, rep_out_dim = utils.train_utils.load_checkpoint_and_model(
+            checkpoint_path, device, 'DAE')
+        self.rep_model = rep_model.to('cpu')
+        self.rep_model.train(False)
+        #   RNN autoencoder
+        checkpoint_path = args.autoencoder_checkpoint
+
+        # RNNAutoencoder_args, RNNAutoencoder_model, RNNAutoencoder_fn,\
+        # RNNAutoencoder_model, RNNAutoencoder_dim = utils.train_utils.load_checkpoint_and_model(
+        #     checkpoint_path, device)
+
+        args_rnn, rnn, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+            checkpoint_path, device, 'autoencoder_vq')
+        self.RNNAutoencoder_model = rnn.to('cpu')
+        self.RNNAutoencoder_model.train(False)
+
+
+    def __len__(self):
+        return self.n_samples
+
+    def __getitem__(self, idx):
+        with self.lmdb_env.begin(write=False) as txn:
+            key = '{:010}'.format(idx).encode('ascii')
+            sample = txn.get(key)
+
+            sample = pyarrow.deserialize(sample)
+            word_seq, pose_seq, audio_raws, audio_mels, aux_info, sentence_leve_latents, GP3_Embedding = sample
+
+        def words_to_tensor(lang, words, end_time=None):
+            # indexes = [lang.SOS_token]
+            indexes = []
+            if len(words)<=0:
+                print()
+            for word in words:
+                if end_time is not None and word[1] > end_time:
+                    break
+                indexes.append(lang.get_word_index(word[0]))
+            # indexes.append(lang.EOS_token)
+            return torch.Tensor(indexes).long()
+
+        def poseIndex_to_tensor(sop, eop, poses_indicies):
+            indexes = [sop]
+            for pose_index in poses_indicies:
+                indexes.append(pose_index)
+            indexes.append(eop)
+            return torch.Tensor(indexes).long()
+
+
+        # normalize
+        std = np.clip(self.data_std, a_min=0.01, a_max=None)
+        pose_seq = (pose_seq - self.data_mean) / std
+
+
+        # to tensors
+        word_seq_tensor = words_to_tensor(self.lang_model, word_seq, aux_info['end_time'])
+        pose_seq = torch.from_numpy(pose_seq).reshape((pose_seq.shape[0], -1)).float()
+
+        sentence_leve_latents = torch.from_numpy(sentence_leve_latents)
+        sentence_leve_latents = sentence_leve_latents.reshape([sentence_leve_latents.shape[0], -1]).float()
+
+        # Todo: we may need do some preprocessing. (e.g. padding, resampling, etc.)
+        # Todo: Move list-->ndarray to the preprocessing.
+        audio_raw_for_now = False
+        if audio_raw_for_now:
+            audio = audio_raws
+        else:
+            audio = audio_mels
+        audio = np.array(audio)
+        audio = torch.from_numpy(audio).float()
+        # audio = torch.reshape(audio, (sentence_leve_latents.shape[0], -1))
+        # audio = torch.reshape(audio, (4, -1))
+
+        # return cluster_id
+        cluster_ids = np.zeros(sentence_leve_latents.shape[0])
+        if self.RNNAutoencoder_model.vq == True:
+            loss_vq, quantized, perplexity_vq, encodings =\
+                self.RNNAutoencoder_model.vq_layer(sentence_leve_latents)
+            cluster_ids = torch.argmax(encodings, dim=1)
+
+            # q = poseIndex_to_tensor(sop=512, eop=513,
+            #                         poses_indicies=cluster_ids.cpu().detach().numpy())
+            # cluster_ids = q
+            # print(cluster_ids)
+        else:
+            cluster_ids = self.kmeanmodel.predict(
+                # self.clustering_transforms['scalar'].transform
+                (sentence_leve_latents.cpu().detach().numpy()))
+            cluster_ids = torch.from_numpy(cluster_ids).long()
+        # cluster_id = torch.Tensor(cluster_id).long()
+        # print(cluster_ids)
+
+        # clusters_portion = self.clustering_transforms['portion'][cluster_ids]
+        # clusters_portion = torch.from_numpy(clusters_portion)
+
+        # print((cluster_ids))
+        try:
+            GP3_Embedding = torch.from_numpy(GP3_Embedding).float()
+        except:
+            pass
+
+        return word_seq_tensor, pose_seq, audio, aux_info, sentence_leve_latents, cluster_ids, GP3_Embedding
+
+    def set_lang_model(self, lang_model):
+        self.lang_model = lang_model
\ No newline at end of file
diff --git a/scripts/inference.py b/scripts/inference.py
new file mode 100644
index 0000000..2347589
--- /dev/null
+++ b/scripts/inference.py
@@ -0,0 +1,189 @@
+import argparse
+import math
+import pickle
+import pprint
+from pathlib import Path
+
+import librosa
+import numpy as np
+import time
+
+import torch
+from scipy.signal import savgol_filter
+import joblib as jl
+
+import utils
+from pymo.preprocessing import *
+from pymo.viz_tools import *
+from pymo.writers import *
+from utils.data_utils import SubtitleWrapper, normalize_string
+from utils.train_utils import set_logger
+
+from data_loader.data_preprocessor import DataPreprocessor
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+
+def generate_gestures(args, pose_decoder, lang_model, words, seed_seq=None):
+    out_list = []
+    clip_length = words[-1][2]
+
+    # pre seq
+    pre_seq = torch.zeros((1, args.n_pre_poses, pose_decoder.pose_dim))
+    if seed_seq is not None:
+        pre_seq[0, :, :] = torch.Tensor(seed_seq[0:args.n_pre_poses])
+    else:
+        mean_pose = args.data_mean
+        mean_pose = torch.squeeze(torch.Tensor(mean_pose))
+        pre_seq[0, :, :] = mean_pose.repeat(args.n_pre_poses, 1)
+
+    # divide into inference units and do inferences
+    unit_time = args.n_poses / args.motion_resampling_framerate
+    stride_time = (args.n_poses - args.n_pre_poses) / args.motion_resampling_framerate
+    if clip_length < unit_time:
+        num_subdivision = 1
+    else:
+        num_subdivision = math.ceil((clip_length - unit_time) / stride_time) + 1
+
+    print('{}, {}, {}, {}'.format(num_subdivision, unit_time, clip_length, stride_time))
+    num_subdivision = min(num_subdivision, 500)  # DEBUG: generate only for the first N divisions
+
+    out_poses = None
+    start = time.time()
+    for i in range(0, num_subdivision):
+        start_time = i * stride_time
+        end_time = start_time + unit_time
+
+        # prepare text input
+        word_seq = DataPreprocessor.get_words_in_time_range(word_list=words, start_time=start_time, end_time=end_time)
+        word_indices = np.zeros(len(word_seq) + 2)
+        word_indices[0] = lang_model.SOS_token
+        word_indices[-1] = lang_model.EOS_token
+        for w_i, word in enumerate(word_seq):
+            print(word[0], end=', ')
+            word_indices[w_i + 1] = lang_model.get_word_index(word[0])
+        print(' ({}, {})'.format(start_time, end_time))
+        in_text = torch.LongTensor(word_indices).unsqueeze(0).to(device)
+
+        # prepare pre seq
+        if i > 0:
+            pre_seq[0, :, :] = out_poses.squeeze(0)[-args.n_pre_poses:]
+        pre_seq = pre_seq.float().to(device)
+
+        # inference
+        words_lengths = torch.LongTensor([in_text.shape[1]])
+        out_poses = pose_decoder(in_text, words_lengths, pre_seq, None)
+        out_seq = out_poses[0, :, :].data.cpu().numpy()
+
+        # smoothing motion transition
+        if len(out_list) > 0:
+            last_poses = out_list[-1][-args.n_pre_poses:]
+            out_list[-1] = out_list[-1][:-args.n_pre_poses]  # delete the last part
+
+            for j in range(len(last_poses)):
+                n = len(last_poses)
+                prev = last_poses[j]
+                next = out_seq[j]
+                out_seq[j] = prev * (n - j) / (n + 1) + next * (j + 1) / (n + 1)
+
+        out_list.append(out_seq)
+
+    print('Avg. inference time: {:.2} s'.format((time.time() - start) / num_subdivision))
+
+    # aggregate results
+    out_poses = np.vstack(out_list)
+
+    return out_poses
+
+
+def main(checkpoint_path, transcript_path):
+    args, generator, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+        checkpoint_path, device, what='baseline')
+    pprint.pprint(vars(args))
+    save_path = '../output/infer_sample'
+    os.makedirs(save_path, exist_ok=True)
+
+    # load lang_model
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    with open(vocab_cache_path, 'rb') as f:
+        lang_model = pickle.load(f)
+
+    # prepare input
+    # Todo: Why? Whas it for the hitmap check?
+    big_infer = False
+    if not big_infer:
+        transcript = SubtitleWrapper(transcript_path).get()
+    else:
+        adress = "/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts/TestSeq00"
+        offset = 0
+        transcript_list = []
+        for i in range(1, 6):
+            current_transcript = SubtitleWrapper(adress + str(i) + ".json").get()
+            for word in current_transcript:
+                word['start_time'] = "{:.2f}".format(offset + float(word['start_time'][0:-1])) + 's'
+                word['end_time'] = "{:.2f}".format(offset + float(word['end_time'][0:-1])) + 's'
+            offset = float(current_transcript[-1]['end_time'][0:-1])
+            transcript_list.append(current_transcript)
+        transcript = [item for subist in transcript_list for item in subist]
+
+    word_list = []
+    for wi in range(len(transcript)):
+        word_s = float(transcript[wi]['start_time'][:-1])
+        word_e = float(transcript[wi]['end_time'][:-1])
+        word = transcript[wi]['word']
+
+        word = normalize_string(word)
+        if len(word) > 0:
+            word_list.append([word, word_s, word_e])
+
+    # inference
+    out_poses = generate_gestures(args, generator, lang_model, word_list)
+
+    # unnormalize
+    mean = np.array(args.data_mean).squeeze()
+    std = np.array(args.data_std).squeeze()
+    std = np.clip(std, a_min=0.01, a_max=None)
+    out_poses = np.multiply(out_poses, std) + mean
+
+    # make a BVH
+    filename_prefix = '{}'.format(transcript_path.stem)
+    make_bvh(save_path, filename_prefix, out_poses)
+
+
+def make_bvh(save_path, filename_prefix, poses):
+    writer = BVHWriter()
+    pipeline = jl.load('../resource/data_pipe.sav')
+
+    # smoothing
+    n_poses = poses.shape[0]
+    out_poses = np.zeros((n_poses, poses.shape[1]))
+
+    for i in range(poses.shape[1]):
+        out_poses[:, i] = savgol_filter(poses[:, i], 15, 2)  # NOTE: smoothing on rotation matrices is not optimal
+
+    # rotation matrix to euler angles
+    out_poses = out_poses.reshape((out_poses.shape[0], -1, 9))
+    out_poses = out_poses.reshape((out_poses.shape[0], out_poses.shape[1], 3, 3))
+    out_euler = np.zeros((out_poses.shape[0], out_poses.shape[1] * 3))
+    for i in range(out_poses.shape[0]):  # frames
+        r = R.from_matrix(out_poses[i])
+        out_euler[i] = r.as_euler('ZXY', degrees=True).flatten()
+
+    bvh_data = pipeline.inverse_transform([out_euler])
+
+    out_bvh_path = os.path.join(save_path, filename_prefix + '_generated.bvh')
+    with open(out_bvh_path, 'w') as f:
+        writer.write(bvh_data[0], f)
+
+
+if __name__ == '__main__':
+    '''
+    ../output/baseline/baseline_model_checkpoint_100.bin
+     /local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts/TestSeq001.json
+    '''
+    parser = argparse.ArgumentParser()
+    parser.add_argument("ckpt_path", type=Path)
+    parser.add_argument("transcript_path", type=Path)
+    args = parser.parse_args()
+
+    main(args.ckpt_path, args.transcript_path)
diff --git a/scripts/inference_Autoencoder.py b/scripts/inference_Autoencoder.py
new file mode 100644
index 0000000..a78d08e
--- /dev/null
+++ b/scripts/inference_Autoencoder.py
@@ -0,0 +1,564 @@
+import argparse
+import math
+import os
+import pickle
+import pprint
+from pathlib import Path
+
+import librosa
+import numpy as np
+import time
+
+import scipy
+import torch
+from scipy.signal import savgol_filter
+import joblib as jl
+
+import utils
+from pymo.preprocessing import *
+from pymo.viz_tools import *
+from pymo.writers import *
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+from tqdm import tqdm
+from utils.data_utils import SubtitleWrapper, normalize_string
+from utils.train_utils import set_logger
+
+from data_loader.data_preprocessor import DataPreprocessor
+
+from trinity_data_to_lmdb import process_bvh
+
+import matplotlib.pyplot as plt
+
+
+from trinity_data_to_lmdb import process_bvh as process_bvh_trinity
+from twh_dataset_to_lmdb import process_bvh as process_bvh_twh
+from twh_dataset_to_lmdb import process_bvh_rot_only as process_bvh_rot_only_twh
+from twh_dataset_to_lmdb import process_bvh_rot_only_Taras as process_bvh_rot_only_Taras
+from twh_dataset_to_lmdb import process_bvh_test1 as process_bvh_rot_test1
+from inference_DAE import feat2bvh, make_bvh_Trinity
+
+from basis_expansions import (Binner,
+                              GaussianKernel,
+                              Polynomial,
+                              LinearSpline,
+                              CubicSpline,
+                              NaturalCubicSpline)
+from sklearn.linear_model import LinearRegression
+import csaps
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+DATASET_TYPE = 'TWH'
+
+def generate_gestures(args, DAE, rnn, bvh_file):
+
+    if DATASET_TYPE == 'Trinity':
+        poses, poses_mirror = process_bvh_trinity(bvh_file)
+    elif DATASET_TYPE == 'TWH':
+        poses = process_bvh_rot_test1(bvh_file)
+        poses_mirror = poses[250:500]
+
+    mean = np.array(args.data_mean).squeeze()
+    std = np.array(args.data_std).squeeze()
+    std = np.clip(std, a_min=0.01, a_max=None)
+    out_poses = (poses_mirror - mean) / std
+
+    target = torch.from_numpy(out_poses)
+    # target = torch.unsqueeze(target,2)
+    target = target.to(device).float()
+    reconstructed = []
+    # for i in range(len(out_poses)):
+    #     input = torch.unsqueeze(target[i],0)
+    #     current_out = pose_decoder(input)
+    #     reconstructed.append(current_out)
+    encoded = DAE.encoder(target)
+
+    # Todo: move this to the input args
+    use_derivitive = False
+    if use_derivitive:
+        diff = [(encoded[n, :] - encoded[n - 1, :]) for n in range(1, encoded.shape[0])]
+        diff.insert(0, torch.zeros_like(encoded[0, :]))
+        encoded = torch.hstack((encoded, torch.stack(diff)))
+    # encoded = torch.squeeze(encoded, 2)
+    # encoded = encoded.to('cpu')
+    # encoded = encoded.detach().numpy()
+    # Todo: remove this.
+
+    all_frames_from_rnn = None
+    decoder_input = None
+    for i in range(0, len(encoded)-args.n_poses, args.n_poses):
+        input_seq = encoded[i:i+args.n_poses]
+        input_pre_seq = encoded[i]
+        output_seq = encoded[i:i+args.n_poses]
+
+        input_seq = torch.unsqueeze(input_seq, 0)
+        input_seq = input_seq.transpose(0, 1)
+
+        output_seq = torch.unsqueeze(output_seq, 0)
+        output_seq = output_seq.transpose(0, 1)
+
+        reconstructed_rnn = torch.zeros(args.n_poses, output_seq.size(1), rnn.decoder.output_size).to(output_seq.device)
+
+        # Todo: For CNN case
+        # reconstructed_rnn = torch.zeros(args.n_poses, output_seq.size(1), 135).to(output_seq.device)
+
+        # run words through encoder
+        encoder_outputs, encoder_hidden = rnn.encoder(input_seq, None)
+        decoder_hidden = encoder_hidden[:rnn.decoder.n_layers]  # use last hidden state from encoder
+
+        if rnn.VAE:
+            decoder_hidden = encoder_hidden[:rnn.decoder.n_layers]  # use last hidden state from encoder
+            # [2, 128, 200]
+            # print("decoder_hidden!!! org", decoder_hidden.shape)
+            decoder_hidden = decoder_hidden.transpose(1, 0).contiguous() # [128, 2, 200]
+            decoder_hidden = torch.reshape(decoder_hidden, (decoder_hidden.shape[0], -1))
+            mean = rnn.VAE_fc_mean(decoder_hidden)
+            logvar = rnn.VAE_fc_std(decoder_hidden)
+            z = rnn.reparameterize(mean, logvar, train=False)
+            z = rnn.VAE_fc_decoder(z)
+            decoder_hidden = z.reshape(decoder_hidden.shape[0],
+                                       rnn.decoder.n_layers, -1)
+            decoder_hidden = decoder_hidden.transpose(1 ,0).contiguous()
+            # print("decoder_hidden!!! modified", decoder_hidden.shape)
+            decoder_first_hidden = decoder_hidden
+        else:
+            decoder_hidden = encoder_hidden[:rnn.decoder.n_layers]  # use last hidden state from encoder
+            min_ = torch.min(encoder_hidden)
+            max_ = torch.max(encoder_hidden)
+            # print("decoder_hidden!!! not VAE ", decoder_hidden.shape)
+
+        if args.autoencoder_vq == 'True':
+            loss_vq, quantized, perplexity_vq, encodings = rnn.vq_layer(decoder_hidden)
+            ifx =  torch.eq(quantized,decoder_hidden)
+
+            decoder_hidden = quantized
+
+        if rnn.CNN:
+            print(decoder_hidden.shape)
+
+            outputs, hidden = rnn.decoder(decoder_hidden)
+            shape = outputs.shape
+            # print(shape)
+            outputs = outputs.transpose(1,2)
+            # print(outputs.shape)
+            shape = outputs.shape
+            outputs = outputs.reshape(shape[0]*shape[1], shape[2])
+            # print(outputs.shape)
+            outputs = rnn.out_layer_decoder(outputs)
+            outputs = outputs.reshape(shape[0], shape[1], -1)
+            reconstructed_rnn = outputs.transpose(0, 1)
+            # print(outputs.shape)
+        else:
+            # run through decoder one time step at a time
+            if decoder_input is None:
+                decoder_input = output_seq[0]  # initial pose from the dataset
+            reconstructed_rnn[0] = decoder_input
+
+            # repeat = 0
+            for rep in range(5):
+                decoder_output, decoder_hidden, _ = rnn.decoder(None, decoder_input, decoder_hidden, encoder_outputs,
+                                                                None)
+
+
+            for t in range(1, rnn.n_frames):
+                if not rnn.autoencoder_conditioned:
+                    decoder_input = torch.zeros_like(decoder_input)
+                decoder_output, decoder_hidden, _ = rnn.decoder(None, decoder_input, decoder_hidden, encoder_outputs,
+                                                                 None)
+                reconstructed_rnn[t] = decoder_output
+
+                if t < rnn.n_pre_poses:
+                    decoder_input = output_seq[t]  # next input is current target
+                else:
+                    decoder_input = decoder_output  # next input is current prediction
+                if not rnn.autoencoder_conditioned:
+                    decoder_input = torch.zeros_like(decoder_input)
+
+
+
+
+        if all_frames_from_rnn == None:
+            all_frames_from_rnn = reconstructed_rnn.transpose(0,1)
+        else:
+            all_frames_from_rnn = torch.cat((all_frames_from_rnn, reconstructed_rnn.transpose(0, 1)), 1)
+
+
+    #     Todo: decode DAE
+    all_frames_from_rnn = torch.squeeze(all_frames_from_rnn, 0)
+    if use_derivitive:
+        all_frames_from_rnn = all_frames_from_rnn[:, 0: all_frames_from_rnn.shape[1]//2]
+    reconstructed_seq_DAE = DAE.decoder(all_frames_from_rnn)
+    # reconstructed_seq_DAE = torch.squeeze(reconstructed_seq_DAE, 2)
+    reconstructed_seq_DAE = reconstructed_seq_DAE.to('cpu')
+    reconstructed_seq_DAE = reconstructed_seq_DAE.detach().numpy()
+
+
+    return out_poses, np.array(reconstructed_seq_DAE)
+
+    '''
+        # smoothing motion transition
+        if len(out_list) > 0:
+            last_poses = out_list[-1][-args.n_pre_poses:]
+            out_list[-1] = out_list[-1][:-args.n_pre_poses]  # delete the last part
+
+            for j in range(len(last_poses)):
+                n = len(last_poses)
+                prev = last_poses[j]
+                next = out_seq[j]
+                out_seq[j] = prev * (n - j) / (n + 1) + next * (j + 1) / (n + 1)
+
+        out_list.append(out_seq)
+
+    print('Avg. inference time: {:.2} s'.format((time.time() - start) / num_subdivision))
+
+    # aggregate results
+    out_poses = np.vstack(out_list)
+    '''
+    return out_poses
+
+Flag_VQ = True
+
+
+def plot_embedding(rnn):
+    import seaborn as sns
+    from sklearn.decomposition import PCA
+    from openTSNE import TSNE
+
+
+    try:
+
+        # Plot Pre linear kernel
+        w = rnn.vq_layer.pre_linear.weight.cpu().detach().numpy()
+        plt.imshow(w, cmap='Reds')
+        plt.show()
+
+
+        w = rnn.vq_layer._embedding.weight.cpu().detach().numpy()
+        # PCA
+
+        # plot embedding as image
+        plt.imshow(w)
+        plt.title("Embeddings as image")
+        plt.show()
+
+        pca = PCA(n_components=50)
+        priciple_components = pca.fit(w)
+        normalized_data = priciple_components.transform(w)
+        # TSNE
+
+        MyTSNE = TSNE(
+            n_components=2,
+            perplexity=30,
+            metric='euclidean',
+            n_jobs=8,
+            verbose=True
+        )
+        X_embedded = MyTSNE.fit(normalized_data)
+
+        plt.figure(figsize=(16, 10))
+        # palette = sns.color_palette("bright", k_component)
+        # 2D
+
+        sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1],
+                        # hue=labels_,
+                        legend=False)
+        plt.title("Embedding visualization")
+        address = os.path.dirname(args.ckpt_path_Autoencode) + '/Embedding.png'
+        plt.savefig(address)
+        plt.show()
+        pass
+    except:
+        pass
+
+def check_prototypes(model):
+    Embeddings = model.vq_layer._embedding
+    weights = Embeddings.weight
+    print(weights.shape)
+    dists = torch.cdist(weights, weights)
+    dists = dists.cpu().detach().numpy()
+    plt.imshow(dists)
+    plt.show()
+
+def main(checkpoint_path_DAE, checkpoint_path_rnn):
+
+    # Frame level autoencoder
+    args, DAE, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+        checkpoint_path_DAE, device, 'DAE')
+
+    # Sequence level autoencoder
+    if not Flag_VQ:
+        args, rnn, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+            checkpoint_path_rnn, device, 'autoencoder')
+    else:
+        args, rnn, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+            checkpoint_path_rnn, device, 'autoencoder_vq')
+        plot_embedding(rnn)
+        if args.autoencoder_vq == 'True':
+            check_prototypes(rnn)
+
+    pprint.pprint(vars(args))
+    save_path = '../output/infer_sample'
+    save_path = os.path.dirname(checkpoint_path_rnn) + '/infer_sample'
+    os.makedirs(save_path, exist_ok=True)
+
+    # load lang_model
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    with open(vocab_cache_path, 'rb') as f:
+        lang_model = pickle.load(f)
+
+    # prepare input
+    # transcript = SubtitleWrapper(transcript_path).get()
+
+    # inference
+    for infer_i in range(1, 2):
+        if DATASET_TYPE == 'Trinity':
+            bvh_file = '/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Motion/TestSeq0'
+            org_poses, reconstructed = generate_gestures(args, DAE, rnn, bvh_file + str(infer_i).zfill(2) + '.bvh')
+        elif DATASET_TYPE == 'TWH':
+            pre_bvh_pth = "/local-scratch/pjomeyaz/rosie_gesture_benchmark/cloned/Clustering/must/GENEA/Co-Speech_Gesture_Generation/dataset/dataset_v1/val/bvh/"
+            bvh_file_names_list = ['val_2022_v1_038.bvh', 'val_2022_v1_012.bvh']
+            org_poses, reconstructed = generate_gestures(args, DAE, rnn,pre_bvh_pth + bvh_file_names_list[infer_i])
+
+        for iq in range(5):
+            plt.plot(reconstructed[:, iq])
+        plt.show()
+        plt.clf()
+        # smoothing motion transition
+        for offset in range(30, len(reconstructed), 30):
+            for index in range(10):
+                n = 10
+                prev = reconstructed[offset - index ]
+                next = reconstructed[offset + index]
+                reconstructed[offset + index] = prev * (n - index) / (n + 1) + next * (index + 1) / (n + 1)
+
+        # upsampling
+
+
+
+
+        # unnormalize
+        mean = np.array(args.data_mean).squeeze()
+        std = np.array(args.data_std).squeeze()
+        std = np.clip(std, a_min=0.01, a_max=None)
+        reconstructed = np.multiply(reconstructed, std) + mean
+        org_poses = np.multiply(org_poses, std) + mean
+
+        # ........ infer
+
+
+        # for i in range(0, len(reconstructed), 30):
+        #     make_bvh(save_path, 'test_' + str(i//30)+ "_f_" + str(i), reconstructed[i:i+30, :])
+
+        # make a BVH
+        filename_prefix = '{}'.format("TestSeq{}".format(infer_i).zfill(3))
+        make_bvh(save_path, filename_prefix, org_poses)
+        filename_prefix = '{}'.format("test_reconstructed_" + str(infer_i))
+        make_bvh(save_path, filename_prefix, reconstructed)
+
+
+def smoothing_function(method, poses):
+
+    if method == 'moving_average':
+        q = np.zeros_like(poses)
+        window_len = 10
+        for i in range(poses.shape[1]):
+            for j in range(poses.shape[0]):
+                neighbours = []
+                for w in range(-window_len, window_len):
+                    if j + w >= 0 and j + w < poses.shape[0]:
+                        neighbours.append(poses[j + w, i])
+
+                q[j, i] = np.array(neighbours).mean()
+        poses = q
+    if method == 'convolution':
+        window_len = 10
+
+        for i in range(poses.shape[1]):
+            a = poses[window_len-1:0:-1, i]
+            b = poses[:, i]
+            c = poses[-1:-window_len:-1, i]
+            s = np.r_[a,b,c]
+
+            w = np.ones(window_len, 'd')
+            y = np.convolve(w / w.sum(), s, mode='valid')
+            zz = y[0:-window_len+1]
+            poses[:, i] = zz
+    if method == 'upsample':
+        q = np.zeros((poses.shape[0] * 2, poses.shape[1]))
+        q = np.zeros((poses.shape[0] * 2, poses.shape[1]))
+        for i in range(poses.shape[1]):
+            q[:, i] = scipy.signal.resample(poses[:, i], poses.shape[0] * 2, domain='time')
+        poses = q
+    if method == 'cubic':
+        win = 10
+        def make_gaussian_regression(n_centers):
+            return Pipeline([
+                ('binner', GaussianKernel(0, 1, n_centers=n_centers, bandwidth=0.1)),
+                ('regression', LinearRegression(fit_intercept=True))
+            ])
+
+        def make_polynomial_regression(degree):
+            return Pipeline([
+                ('std', StandardScaler()),
+                ('poly', Polynomial(degree=degree)),
+                ('regression', LinearRegression(fit_intercept=True))
+            ])
+        x = np.array(range(0, win))
+        for offset in range(0, poses.shape[0]-win, win):
+            for joint in range(poses.shape[1]):
+                gr = make_gaussian_regression(1)
+                gr.fit(x, poses[offset:offset+win, joint])
+                plt.scatter(x, poses[offset:offset+win, joint], color='red')
+                plt.scatter(x, gr.predict(x), color='blue')
+                plt.plot(poses[offset:offset + win, joint], color='red')
+                plt.plot(gr.predict(x), color='blue')
+                poses[offset:offset+win, joint] = gr.predict(x)
+                # plt.show()
+    if method == 'spline':
+
+        # deriv = []
+        # for i in range(2, poses.shape[0]):
+        #     deriv.append(np.sum(np.abs(poses[i] - poses[i-1])))
+        # plt.plot(deriv)
+        # plt.title("Derivitive")
+        # # plt.show()
+
+        win =poses.shape[0]
+        upsample_in = False
+        smooth_f = 0.5
+        x = np.array(range(0, win))
+        x_up = np.linspace(0, win, win*2, endpoint=True)
+        if upsample_in:
+            q = np.zeros((poses.shape[0] * 2, poses.shape[1]))
+        else:
+            q = np.zeros((poses.shape[0], poses.shape[1]))
+        test = range(0, poses.shape[0] - win, win//4)
+        for offset in tqdm(range(0, poses.shape[0], win)):
+            if offset == 0:
+                print("if offset == 330:")
+            for joint in range(poses.shape[1]):
+                # gr = csaps.UnivariateCubicSmoothingSpline(x, poses[offset:offset + win, joint],
+                #                                           smooth=0.85)
+                # predicted = csaps.csaps(x, poses[offset:offset + win, joint], x, smooth=-3)
+                if upsample_in:
+                    predicted = csaps.csaps(x, poses[:, joint], x_up, smooth=smooth_f)
+                    predicted = csaps.csaps(x, predicted, x_up, smooth=smooth_f)
+
+                else:
+                    predicted = csaps.csaps(x, poses[:, joint], x, smooth=smooth_f)
+
+                q[offset:offset + win * 2, joint] = (predicted)
+
+                # if offset==0 and joint==0:
+                #     # print()
+                #     # plt.scatter(x, poses[offset:offset + win, joint], color='red')
+                #     # plt.scatter(x, predicted, color='blue')
+                #     plt.plot(poses[offset:offset + win, joint], color='red')
+                #     plt.plot(predicted, color='blue')
+                #     plt.title("smooth="+str(smooth_f))
+                #     # plt.show()
+            # exit()
+            poses = q
+
+    # deriv = []
+    # for i in range(2, poses.shape[0]):
+    #     deriv.append(np.sum(np.abs(poses[i] - poses[i - 1])))
+    # plt.plot(deriv)
+    # plt.title("Derivitive_after")
+    # # plt.show()
+
+    return poses
+
+
+
+
+
+def make_bvh(save_path, filename_prefix, poses):
+
+    if DATASET_TYPE=='TWH':
+        return make_bvh_Trinity(save_path, filename_prefix, poses)
+        return feat2bvh(save_path, filename_prefix, poses)
+
+    writer = BVHWriter()
+    pipeline = jl.load('../resource/data_pipe.sav')
+
+    # smoothing
+    n_poses = poses.shape[0]
+    out_poses = np.zeros((n_poses, poses.shape[1]))
+
+
+    # out_poses = np.zeros_like(poses)
+    for i in range(poses.shape[1]):
+        out_poses[:, i] = savgol_filter(poses[:, i], 15, polyorder=3)  # NOTE: smoothing on rotation matrices is not optimal
+
+
+
+    # rotation matrix to euler angles
+    out_poses = out_poses.reshape((out_poses.shape[0], -1, 9))
+    out_poses = out_poses.reshape((out_poses.shape[0], out_poses.shape[1], 3, 3))
+    out_euler = np.zeros((out_poses.shape[0], out_poses.shape[1] * 3))
+    for i in range(out_poses.shape[0]):  # frames
+        r = R.from_matrix(out_poses[i])
+        out_euler[i] = r.as_euler('ZXY', degrees=True).flatten()
+
+    out_euler = smoothing_function('spline',out_euler)
+
+    bvh_data = pipeline.inverse_transform([out_euler])
+
+    out_bvh_path = os.path.join(save_path, filename_prefix + '_generated.bvh')
+    with open(out_bvh_path, 'w') as f:
+        writer.write(bvh_data[0], f)
+
+def plot_loss(checkpoint_path_rnn):
+
+    x = torch.load(checkpoint_path_rnn)
+    all_eval_loss = x['val_metrics_list']
+    all_train_loss = x['loss_list']
+    # X = np.arange(136-3)
+    # fig = plt.figure()
+    # ax = fig.add_axes([0, 0, 1, 1])
+    # ax.bar(X + 0.00, all_train_loss[0], color='b', width=0.25)
+    # ax.bar(X + 0.25, all_eval_loss[1], color='g', width=0.25)
+    # plt.show()
+    # plotting the second plot
+    plt.plot(all_train_loss, label='Train loss')
+    plt.plot(all_eval_loss, label='Evaluation loss')
+
+    # Labeling the X-axis
+    plt.xlabel('Epoch number')
+    # Labeling the Y-axis
+    plt.ylabel('Loss Average')
+    # Give a title to the graph
+    plt.title('Training/Evaluation Loss based on epoch number')
+
+    # Show a legend on the plot
+    plt.legend()
+
+    address = os.path.dirname(args.ckpt_path_Autoencode) + '/loss_plot.png'
+    plt.savefig(address)
+    # plt.savefig(os.path.join(args.model_save_path, 'loss_plot.png'))
+    plt.show()
+    # exit()
+
+if __name__ == '__main__':
+    '''
+    
+        ../output/DAE/train_DAE_H41/rep_learning_DAE_H41_checkpoint_020.bin
+        ../output/autoencoder/without_att_fxw_zinput/autoencode_fxw_zinput_checkpoint_100.bin
+        
+        
+        ../output/DAE/train_DAE_H41/rep_learning_DAE_H41_checkpoint_020.bin
+        ../output/autoencoder/VAE/Dautoencode_fxw_zinput_checkpoint_100.bin
+        
+        
+        ../output/DAE/train_DAE_H41/rep_learning_DAE_H41_checkpoint_020.bin
+../output/autoencoder/VAE+sim/autoencoder_decoder_trained_cont/autoencoder_decoder_trained_cont_checkpoint_040.bin
+    '''
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("ckpt_path_DAE", type=Path)
+    parser.add_argument("ckpt_path_Autoencode", type=Path)
+    args = parser.parse_args()
+
+    plot_loss(args.ckpt_path_Autoencode)
+
+    main(args.ckpt_path_DAE, args.ckpt_path_Autoencode)
diff --git a/scripts/inference_DAE.py b/scripts/inference_DAE.py
new file mode 100644
index 0000000..ee5c006
--- /dev/null
+++ b/scripts/inference_DAE.py
@@ -0,0 +1,659 @@
+import argparse
+import math
+import os
+import pickle
+import pprint
+from pathlib import Path
+
+import librosa
+import numpy as np
+import time
+
+import torch
+from scipy.signal import savgol_filter
+import joblib as jl
+
+import utils
+from pymo.preprocessing import *
+from pymo.preprocessing import MocapParameterizer
+from pymo.viz_tools import *
+from pymo.writers import *
+from tqdm import tqdm
+from utils.data_utils import SubtitleWrapper, normalize_string
+from utils.train_utils import set_logger
+
+from data_loader.data_preprocessor import DataPreprocessor
+
+from trinity_data_to_lmdb import process_bvh as process_bvh_trinity
+from twh_dataset_to_lmdb import process_bvh as process_bvh_twh
+from twh_dataset_to_lmdb import process_bvh_rot_only as process_bvh_rot_only_twh
+from twh_dataset_to_lmdb import process_bvh_rot_only_Taras as process_bvh_rot_only_Taras
+from twh_dataset_to_lmdb import process_bvh_test1 as process_bvh_rot_test1
+
+from sklearn.decomposition import PCA
+from openTSNE import TSNE
+import seaborn as sns
+
+import matplotlib.pyplot as plt
+from scipy.ndimage.filters import gaussian_filter
+from sklearn.preprocessing import StandardScaler
+
+# assignment4 imports
+import torch
+import torchvision
+import torchvision.transforms as transforms
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+from torchvision.utils import save_image
+from tqdm import tqdm
+import numpy as np
+
+import matplotlib.pyplot as plt
+import imageio
+import matplotlib.image as mpimg
+from scipy import ndimage
+
+from sklearn import pipeline
+from sklearn.cluster import DBSCAN, KMeans, AgglomerativeClustering
+
+from model.DAE_model import DAE_Network, VQ_Frame
+from sklearn.preprocessing import MinMaxScaler
+from sklearn.metrics import silhouette_samples, silhouette_score, pairwise_distances
+DATASET_Type = 'Trinity'
+DATASET_Type = 'TWH'
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+# device = 'cpu'
+
+def generate_gestures(args, pose_decoder, bvh_file):
+
+    if DATASET_Type == 'Trinity':
+        poses, poses_mirror = process_bvh_trinity(bvh_file)
+    elif DATASET_Type == 'TWH':
+        poses = process_bvh_rot_test1(bvh_file)
+        poses_mirror = poses[250:600]
+    mean = np.array(args.data_mean).squeeze()
+    std = np.array(args.data_std).squeeze()
+    std = np.clip(std, a_min=0.01, a_max=None)
+    out_poses = (poses_mirror - mean) / std
+
+    target = torch.from_numpy(out_poses)
+    target = torch.unsqueeze(target,2)
+    target = target.to(device).float()
+    reconstructed = []
+    # for i in range(len(out_poses)):
+    #     input = torch.unsqueeze(target[i],0)
+    #     current_out = pose_decoder(input)
+    #     reconstructed.append(current_out)
+    if args.autoencoder_vq == 'True':
+        check_prototypes(pose_decoder)
+        reconstructed, latent, encodings = pose_decoder(target, Inference=True)
+    else:
+        reconstructed, latent = pose_decoder(target, get_latent=True)
+    reconstructed = torch.squeeze(reconstructed, 2)
+    reconstructed = reconstructed.to('cpu')
+    reconstructed = reconstructed.detach().numpy()
+
+
+
+    if latent is not None: # for H-1 case
+        latent = latent.cpu().detach().numpy()
+
+    if args.autoencoder_vq == 'True':
+        encodings = encodings.detach().cpu().numpy()
+        return out_poses, np.array(reconstructed), latent, encodings
+    else:
+        return out_poses, np.array(reconstructed), latent
+    '''
+        # smoothing motion transition
+        if len(out_list) > 0:
+            last_poses = out_list[-1][-args.n_pre_poses:]
+            out_list[-1] = out_list[-1][:-args.n_pre_poses]  # delete the last part
+
+            for j in range(len(last_poses)):
+                n = len(last_poses)
+                prev = last_poses[j]
+                next = out_seq[j]
+                out_seq[j] = prev * (n - j) / (n + 1) + next * (j + 1) / (n + 1)
+
+        out_list.append(out_seq)
+
+    print('Avg. inference time: {:.2} s'.format((time.time() - start) / num_subdivision))
+
+    # aggregate results
+    out_poses = np.vstack(out_list)
+    '''
+    return out_poses
+
+def check_prototypes(model):
+    Embeddings = model.vq_layer._embedding
+    weights = Embeddings.weight
+    print(weights.shape)
+    dists = torch.cdist(weights, weights)
+    dists = dists.cpu().detach().numpy()
+    plt.imshow(dists)
+    plt.show()
+
+def k_components_analysis_KMEANS(latent):
+
+    mms = MinMaxScaler()
+    mms.fit(latent)
+    data_transformed = mms.transform(latent)
+
+    Sum_of_squared_distances = []
+    silhouette_scores = []
+    K = range(20, 100, 1)
+    for k in tqdm(K):
+        km = KMeans(n_clusters=k, max_iter=2500,
+                        random_state=0)
+        km = km.fit(data_transformed)
+        Sum_of_squared_distances.append(km.inertia_)
+        s_score = silhouette_score(data_transformed, km.labels_)
+        silhouette_scores.append(s_score)
+
+
+    plt.plot(K, Sum_of_squared_distances, 'bx-')
+    plt.xlabel('k')
+    plt.ylabel('Sum_of_squared_distances')
+    plt.title('Elbow Method For Optimal k')
+    plt.show()
+
+    plt.plot(K, silhouette_scores, 'bx-')
+    plt.xlabel('k')
+    plt.ylabel('Silhouette Score')
+    plt.title('Silhouette Method For Optimal k')
+    plt.show()
+
+    exit()
+
+def k_components_analysis_VQ(base_folder, mink, maxk):
+
+    Sum_of_squared_distances = []
+    silhouette_scores = []
+
+    for model_id in range(mink, maxk):
+        checkpoint_path = base_folder + '/' +\
+                          str(model_id) + '/DAE_H40_checkpoint_030.bin'
+        # 1. load model
+        args, generator, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+            checkpoint_path, device, 'DAE')
+
+
+        bvh_file = '/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Motion/TestSeq001.bvh'
+        org_poses, reconstructed, latent, encodings = generate_gestures(args, generator, bvh_file)
+
+        # Generate latents
+        indices = np.argmax(encodings, axis=1)
+        centers = generator.vq_layer._embedding.weight.cpu().detach().numpy()
+        k_component = generator.vq_components
+
+
+        mms = MinMaxScaler()
+        mms.fit(latent)
+        data_transformed = mms.transform(latent)
+
+
+
+
+        # Sum_of_squared_distances.append(km.inertia_)
+        s_score = silhouette_score(data_transformed, indices)
+        silhouette_scores.append(s_score)
+
+
+        # plt.plot(K, Sum_of_squared_distances, 'bx-')
+        # plt.xlabel('k')
+        # plt.ylabel('Sum_of_squared_distances')
+        # plt.title('Elbow Method For Optimal k')
+        # plt.show()
+
+    plt.plot(range(mink, maxk), silhouette_scores, 'bx-')
+    plt.xlabel('k')
+    plt.ylabel('Silhouette Score')
+    plt.title('Silhouette Method For Optimal k')
+    plt.show()
+
+    exit()
+
+
+def Save4Unity(latents, indecies, components, kernels, save_path):
+    # Save for unity:
+
+    # 0. prep
+    f = open(save_path + '/latents.txt', 'w')
+    f.write(str(components) + '\n')
+
+    # Fitting transformations
+    # pca = PCA(n_components=20)
+    # priciple_components = pca.fit(latents)
+
+
+
+    MyTSNE = TSNE(
+        n_components=2,
+        perplexity=200,
+        metric='euclidean',
+        n_jobs=8,
+        verbose=True,
+        n_iter=5000, early_exaggeration_iter=300,
+    )
+    combined = np.concatenate((kernels, latents), axis=0)
+    TSNE_Transfotm = MyTSNE.fit(combined)
+
+    transformed_kernels = TSNE_Transfotm[0:len(kernels)]
+    transformed_latents = TSNE_Transfotm[len(kernels):]
+
+    # 1. Embeddings:
+    for i in range(0, len(transformed_kernels)):
+        line =  "{:.3f}".format(transformed_kernels[i, 0]) + ','\
+                +  "{:.3f}".format(transformed_kernels[i, 1])
+        f.write(line + '\n')
+
+    # 2. latents
+    for i in range(0, len(transformed_latents)):
+        line = str(i) + ',' "{:.3f}".format(transformed_latents[i, 0]) + ','\
+               + "{:.3f}".format(transformed_latents[i, 1]) + \
+                                    ',' + str(indecies[i])
+        f.write(line + '\n')
+
+
+
+
+def plot_latent(args, latents, model):
+
+    # TSNE
+
+    MyTSNE = TSNE(
+        n_components=2,
+        perplexity=30,
+        metric='euclidean',
+        n_jobs=8,
+        verbose=True
+    )
+    X_embedded = MyTSNE.fit(latents)
+
+
+
+    plt.figure(figsize=(16, 10))
+    # 2D
+    sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1],
+                     legend=False, palette=sns.color_palette('Set2'))
+
+    address = (args.model_save_path) + '/'
+    os.makedirs(address, exist_ok=True)
+    address += 'Embedding_latent.png'
+    plt.savefig(address)
+
+    plt.show()
+
+
+
+    # Save for unity:
+
+
+
+    # Heatmap
+    x_lim, y_lim = 60, 60
+
+    heatmap, x_edge, y_edge = np.histogram2d(X_embedded[:, 0], X_embedded[:, 1], bins=100)
+    extent = [x_edge[0], x_edge[-1], y_edge[0], y_edge[-1]]
+    extent = [-y_lim, y_lim, -x_lim, x_lim]
+    plt.clf()
+    sigma = 2
+    for sigma in range(1, 3):
+        plt.figure(figsize=(16, 10))
+        heatmap_filtered = gaussian_filter(heatmap, sigma)
+        plt.imshow(heatmap_filtered.T, extent=extent, origin='lower')
+        # plt.colorbar(plt.pcolor(heatmap))
+        # plt.pcolor(heatmap)
+        plt.colorbar()
+        plt.title('Heatmap of representation dataset, Sigma = ' + str(sigma))
+        plt.show()
+        plt.clf()
+
+    w = model.encoder[0].weight
+    w = w.cpu().detach().numpy()
+
+    scaler = StandardScaler()
+    scaler.fit(w)
+    # Todo: check which one is better? Unnormalized showed better results.
+    normalized_data = scaler.transform(w)
+
+    max = np.max(normalized_data)
+    min = np.min(normalized_data)
+
+    plt.imshow(w)
+    plt.title("Kernel w ({},{})".
+              format(np.min(w), np.max(w)))
+    plt.show()
+
+    if isinstance(model, VQ_Frame):
+        if model.vae:
+            #Plot Variance kernel
+            w = model.VAE_fc_std.weight
+            w = w.cpu().detach().numpy()
+            plt.imshow(w)
+            plt.title("Kernel of STD layer ({},{})".
+                      format(np.min(w), np.max(w)))
+            plt.show()
+
+            # Plot Variance kernel
+            w = model.VAE_fc_mean.weight
+            w = w.cpu().detach().numpy()
+            plt.imshow(w)
+            plt.title("Kernel of Mean layer ({},{})".
+                      format(np.min(w), np.max(w)))
+            plt.show()
+
+    return
+    plt.imshow(normalized_data)
+    plt.title("Kernel mormalized ({},{})".
+              format(np.min(normalized_data), np.max(normalized_data)))
+    plt.show()
+
+    q = (w - np.min(w))/(np.max(w)-np.min(w))
+    plt.imshow(q)
+    plt.title("Kernel q ({},{})".
+              format(np.min(q), np.max(q)))
+    plt.show()
+
+
+def Plot_Kernel(_model, args):
+    '''
+    the plot for visualizing the learned weights of the autoencoder's encoder .
+    ----------
+    _model: Autoencoder
+    '''
+    # needs your implementation
+
+    def to_eular(input_pose, args):
+        # unnormalize
+        mean = np.array(args.data_mean).squeeze()
+        std = np.array(args.data_std).squeeze()
+        std = np.clip(std, a_min=0.01, a_max=None)
+        input_pose = np.multiply(input_pose, std) + mean
+
+
+        # rotation matrix to euler angles
+        out_poses = input_pose.reshape((-1, 9))
+        out_poses = out_poses.reshape((out_poses.shape[0], 3, 3))
+        out_euler = np.zeros(( out_poses.shape[0] * 3))
+
+        r = R.from_matrix(out_poses)
+        out_euler = r.as_euler('ZXY', degrees=True).flatten()
+
+        pipeline = jl.load('../resource/data_pipe.sav')
+        out_euler = np.expand_dims(out_euler, axis=0)
+        bvh_data = pipeline.inverse_transform([out_euler])
+        return bvh_data
+        return out_euler.reshape((15,3))
+    w = _model.encoder[0].weight
+    w = w.cpu().detach().numpy()
+
+    # scaler = StandardScaler()
+    # scaler.fit(w)
+    # Todo: check which one is better? Unnormalized showed better results.
+
+    plt.imshow(w)
+    plt.title("Kernel w ({},{})".
+              format(np.min(w), np.max(w)))
+    plt.show()
+
+    for i in range(len(w)):
+        bvh_data = to_eular(w[i], args)
+        q = MocapParameterizer('position')
+        # dr_pipe = pipeline([
+        #     ('param', q),
+        # ])
+        XX = q.transform(bvh_data)
+        ax = draw_stickfigure(XX[0], 0, XX[0].values)
+        plt.show()
+
+        plt.imshow(w[i].reshape((15,9)))
+        plt.title("15,9 style")
+        plt.show()
+
+        # plt.scatter(eu[:,2], eu[:,1])
+        # plt.title("Human shape")
+        # plt.show()
+
+
+
+
+
+
+
+
+
+
+    # exit()
+    pass
+
+
+
+def main(checkpoint_path):
+    args, generator, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+        checkpoint_path, device, 'DAE')
+    pprint.pprint(vars(args))
+    save_path = os.path.dirname(checkpoint_path)
+    os.makedirs(save_path, exist_ok=True)
+
+    # load lang_model
+    # vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    # with open(vocab_cache_path, 'rb') as f:
+    #     lang_model = pickle.load(f)
+
+    # prepare input
+    # transcript = SubtitleWrapper(transcript_path).get()
+
+    # inference
+    # 1. Trinity Dataset
+    if DATASET_Type == 'Trinity':
+        bvh_file = '/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Motion/TestSeq001.bvh'
+    # 2. Talking With Hands 16.2M
+    elif DATASET_Type == 'TWH':
+        bvh_file = '/local-scratch/pjomeyaz/rosie_gesture_benchmark/cloned/Clustering/must/GENEA/Co-Speech_Gesture_Generation/dataset/dataset_v1/val/bvh/val_2022_v1_012.bvh'
+
+    if args.autoencoder_vq=='True':
+        org_poses, reconstructed, latent, encodings = generate_gestures(args, generator, bvh_file)
+    else:
+        org_poses, reconstructed, latent = generate_gestures(args, generator, bvh_file)
+
+    # Plot_Kernel(_model=generator, args=args)
+    try:
+        plot_latent(args, latent, generator)
+    except Exception as e:
+        print("Exception in plot_latent", e)
+
+    if False:
+        if isinstance(generator, VQ_Frame):
+            indices = np.argmax(encodings, axis=1)
+            centers = generator.vq_layer._embedding.weight.cpu().detach().numpy()
+            k_component = generator.vq_components
+        else: #     kmeans algorithm
+            k_component = 100
+            km_rnn = KMeans(n_clusters=k_component, max_iter=2500,
+                            random_state=0).fit(latent)
+            print("Kmeans trained!")
+            indices = km_rnn.labels_
+            # centers = generator.encoder[0].weight.cpu().detach().numpy()
+            centers = km_rnn.cluster_centers_
+
+    # k_components_analysis_KMEANS(latent)
+
+    # Save4Unity(latent, indices, k_component, centers, save_path)
+
+    # unnormalize
+    mean = np.array(args.data_mean).squeeze()
+    std = np.array(args.data_std).squeeze()
+    std = np.clip(std, a_min=0.01, a_max=None)
+    reconstructed = np.multiply(reconstructed, std) + mean
+    org_poses = np.multiply(org_poses, std) + mean
+
+    if False:
+        # if args.autoencoder_vq == 'True':
+        #     indices = np.argmax(encodings,axis=1)
+        for frame, index in tqdm(enumerate(indices)):
+            filename_prefix = '{}'.format(frame)
+            make_bvh(save_path + '/clusters/' + str(index), filename_prefix,
+                     np.expand_dims(org_poses[frame],axis=0))
+            filename_prefix = '{}'.format('Rec_of_emb_' + str(index))
+            make_bvh(save_path + '/clusters/' + str(index), filename_prefix,
+                     np.expand_dims(reconstructed[frame], axis=0))
+
+    # ........ infer
+
+
+
+    # make a BVH
+    filename_prefix = '{}'.format("test_original_DAE")
+    make_bvh(save_path, filename_prefix, org_poses)
+    filename_prefix = '{}'.format("test_reconstructed_DAE")
+    make_bvh(save_path, filename_prefix, reconstructed)
+
+
+def make_bvh(save_path, filename_prefix, poses):
+    # return feat2bvh(save_path, filename_prefix, poses)
+    return make_bvh_Trinity(save_path, filename_prefix, poses)
+    if DATASET_Type == 'Trinity':
+        return make_bvh_Trinity(save_path, filename_prefix, poses)
+    elif DATASET_Type == 'TWH':
+        make_bvh_TWH(save_path, filename_prefix, poses)
+
+def make_bvh_Trinity(save_path, filename_prefix, poses):
+    writer = BVHWriter()
+    pipeline = jl.load('../resource/data_pipe.sav')
+
+    # smoothing
+    n_poses = poses.shape[0]
+    out_poses = np.zeros((n_poses, poses.shape[1]))
+
+    if n_poses> 15:
+        for i in range(poses.shape[1]):
+            out_poses[:, i] = savgol_filter(poses[:, i], 15, 2)  # NOTE: smoothing on rotation matrices is not optimal
+    else:
+        out_poses = poses
+
+    # rotation matrix to euler angles
+    out_poses = out_poses.reshape((out_poses.shape[0], -1, 9))   # (n_frames, n_joints, 9)
+    out_poses = out_poses.reshape((out_poses.shape[0], out_poses.shape[1], 3, 3))
+    out_euler = np.zeros((out_poses.shape[0], out_poses.shape[1] * 3))
+    for i in range(out_poses.shape[0]):  # frames
+        r = R.from_matrix(out_poses[i])
+        out_euler[i] = r.as_euler('ZXY', degrees=True).flatten()
+
+    bvh_data = pipeline.inverse_transform([out_euler])
+
+
+    out_bvh_path = os.path.join(save_path, filename_prefix + '.bvh')
+    if not os.path.exists(save_path):
+        os.makedirs(save_path)
+    with open(out_bvh_path, 'w') as f:
+        writer.write(bvh_data[0], f)
+
+
+
+
+
+def make_bvh_TWH(save_path, filename_prefix, poses):
+    writer = BVHWriter()
+    pipeline = jl.load('../resource/data_pipe_TWH.sav')
+
+    # smoothing
+    n_poses = poses.shape[0]
+    out_poses = np.zeros((n_poses, poses.shape[1]))
+
+    for i in range(poses.shape[1]):
+        out_poses[:, i] = savgol_filter(poses[:, i], 15, 2)  # NOTE: smoothing on rotation matrices is not optimal
+
+    # rotation matrix to euler angles
+    out_poses = out_poses.reshape((out_poses.shape[0], -1, 12))  # (n_frames, n_joints, 12)
+    out_data = np.zeros((out_poses.shape[0], out_poses.shape[1], 6))
+    for i in range(out_poses.shape[0]):  # frames
+        for j in range(out_poses.shape[1]):  # joints
+            out_data[i, j, :3] = out_poses[i, j, :3]
+            r = R.from_matrix(out_poses[i, j, 3:].reshape(3, 3))
+            out_data[i, j, 3:] = r.as_euler('ZXY', degrees=True).flatten()
+
+    out_data = out_data.reshape(out_data.shape[0], -1)
+    bvh_data = pipeline.inverse_transform([out_data])
+
+    out_bvh_path = os.path.join(save_path, filename_prefix + '_generated.bvh')
+    with open(out_bvh_path, 'w') as f:
+        writer.write(bvh_data[0], f)
+
+
+def feat2bvh(save_path, filename_prefix, poses):
+    writer = BVHWriter()
+    pipeline = jl.load('../resource/data_pipe.sav')
+
+
+    # transform the data back to it's original shape
+    # note: in a real scenario this is usually done with predicted data
+    # note: some transformations (such as transforming to joint positions) are not inversible
+    poses = pipeline.inverse_transform([poses])
+
+    # ensure correct body orientation
+    # poses[0].values["body_world_Xrotation"] = 0
+    # poses[0].values["body_world_Yrotation"] = 0
+    # poses[0].values["body_world_Zrotation"] = 0
+
+    # Test to write some of it to file for visualization in blender or motion builder
+    out_bvh_path = os.path.join(save_path, filename_prefix + '.bvh')
+    with open(out_bvh_path, 'w') as f:
+        writer.write(poses[0], f)
+
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("ckpt_path", type=Path)
+    args = parser.parse_args()
+
+
+    # k_components_analysis_VQ('../output/DAE_New/VQs', 5, 200)
+
+    main(args.ckpt_path)
+
+
+
+
+def scatter_plot(latent_representations, labels):
+    '''
+    the scatter plot for visualizing the latent representations with the ground truth class label
+    ----------
+    latent_presentations: (N, dimension_latent_representation)
+    labels: (N, )  the labels of the ground truth classes
+    '''
+
+    # borrowed from https://gist.github.com/jakevdp/91077b0cae40f8f8244a
+
+    # Note that if the dimension_latent_representation > 2 you need to apply TSNE transformation
+    # to map the latent representations from higher dimensionality to 2D
+    # You can use #from sklearn.manifold import TSNE#
+
+    def discrete_cmap(n, base_cmap=None):
+        """Create an N-bin discrete colormap from the specified input map"""
+        base = plt.cm.get_cmap(base_cmap)
+        return base.from_list(base.name + str(n), base(np.linspace(0, 1, n)), n)
+
+    plt.figure(figsize=(10, 10))
+    plt.scatter(latent_representations[:, 0], latent_representations[:, 1], cmap=discrete_cmap(10, 'jet'), c=labels,
+                edgecolors='black')
+    plt.colorbar()
+    plt.grid()
+    plt.show()
+
+
+
+def display_images_in_a_row(images, file_path='./tmp.png', display=True):
+    '''
+    images: (N,28,28): N images of 28*28 as a numpy array
+    file_path: file path name for where to store the figure
+    display: display the image or not
+    '''
+    save_image(images.view(-1, 1, 28, 28),
+               '{}'.format(file_path))
+    if display is True:
+        plt.imshow(mpimg.imread('{}'.format(file_path)))
+
+
diff --git a/scripts/inference_cluster2gesture.py b/scripts/inference_cluster2gesture.py
new file mode 100644
index 0000000..51746cb
--- /dev/null
+++ b/scripts/inference_cluster2gesture.py
@@ -0,0 +1,215 @@
+import argparse
+import math
+import pickle
+import pprint
+from pathlib import Path
+
+import librosa
+import numpy as np
+import time
+
+import torch
+from scipy.signal import savgol_filter
+import joblib as jl
+
+import utils
+from pymo.preprocessing import *
+from pymo.viz_tools import *
+from pymo.writers import *
+from tqdm import tqdm
+from utils.data_utils import SubtitleWrapper, normalize_string
+from utils.train_utils import set_logger
+
+from data_loader.data_preprocessor import DataPreprocessor
+
+from trinity_data_to_lmdb import process_bvh
+
+import matplotlib.pyplot as plt
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+
+def generate_gestures(args, DAE, rnn, bvh_file):
+    poses, poses_mirror = process_bvh(bvh_file)
+
+    mean = np.array(args.data_mean).squeeze()
+    std = np.array(args.data_std).squeeze()
+    std = np.clip(std, a_min=0.01, a_max=None)
+    out_poses = (poses_mirror - mean) / std
+
+    target = torch.from_numpy(out_poses)
+    # target = torch.unsqueeze(target,2)
+    target = target.to(device).float()
+    reconstructed = []
+    # for i in range(len(out_poses)):
+    #     input = torch.unsqueeze(target[i],0)
+    #     current_out = pose_decoder(input)
+    #     reconstructed.append(current_out)
+    encoded = DAE.encoder(target)
+
+    # Todo: move this to the input args
+    use_derivitive = True
+    if use_derivitive:
+        diff = [(encoded[n, :] - encoded[n - 1, :]) for n in range(1, encoded.shape[0])]
+        diff.insert(0, torch.zeros_like(encoded[0, :]))
+        encoded = torch.hstack((encoded, torch.stack(diff)))
+    # encoded = torch.squeeze(encoded, 2)
+    # encoded = encoded.to('cpu')
+    # encoded = encoded.detach().numpy()
+    final_result = {}
+    n_sample = 20
+    for i in range(0, 40):
+        input_seq = torch.from_numpy(np.repeat(i, n_sample)).to(device)
+        input_pre_seq = encoded[i]
+        output_seq = encoded[i:i + args.n_poses]
+        output_seq = torch.cat([encoded[i:i + args.n_poses].unsqueeze(0) for rep in range(n_sample)])
+
+        # run through decoder one time step at a time
+        output_current = rnn(input_seq, output_seq)
+        if use_derivitive:
+            output_current = output_current[:, :, : output_current.shape[2] // 2]
+        saved_shaped = output_current.shape
+        output_current = torch.reshape(output_current, (saved_shaped[0]*saved_shaped[1], saved_shaped[2]))
+        reconstructed_seq_DAE = DAE.decoder(output_current)
+        reconstructed_seq_DAE = torch.reshape(reconstructed_seq_DAE, (saved_shaped[0], saved_shaped[1], -1))
+        reconstructed_seq_DAE = reconstructed_seq_DAE.to('cpu')
+        reconstructed_seq_DAE = reconstructed_seq_DAE.detach().numpy()
+        final_result[i] = np.array(reconstructed_seq_DAE)
+
+
+    return final_result
+
+    '''
+        # smoothing motion transition
+        if len(out_list) > 0:
+            last_poses = out_list[-1][-args.n_pre_poses:]
+            out_list[-1] = out_list[-1][:-args.n_pre_poses]  # delete the last part
+
+            for j in range(len(last_poses)):
+                n = len(last_poses)
+                prev = last_poses[j]
+                next = out_seq[j]
+                out_seq[j] = prev * (n - j) / (n + 1) + next * (j + 1) / (n + 1)
+
+        out_list.append(out_seq)
+
+    print('Avg. inference time: {:.2} s'.format((time.time() - start) / num_subdivision))
+
+    # aggregate results
+    out_poses = np.vstack(out_list)
+    '''
+    return out_poses
+
+
+def main(checkpoint_path_DAE, checkpoint_path_rnn):
+    args, DAE, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+        checkpoint_path_DAE, device)
+    args, rnn, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+        checkpoint_path_rnn, device, 'c2g')
+    pprint.pprint(vars(args))
+    save_path = '../output/infer_sample'
+    os.makedirs(save_path, exist_ok=True)
+
+    # load lang_model
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    with open(vocab_cache_path, 'rb') as f:
+        lang_model = pickle.load(f)
+
+    # prepare input
+    # transcript = SubtitleWrapper(transcript_path).get()
+
+    # inference
+    bvh_file = '/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Motion/TestSeq001.bvh'
+
+    reconstructed = generate_gestures(args, DAE, rnn, bvh_file)
+
+    # unnormalize
+    mean = np.array(args.data_mean).squeeze()
+    std = np.array(args.data_std).squeeze()
+    std = np.clip(std, a_min=0.01, a_max=None)
+    for i_cluster in range(0, 40):
+        for j in range(0, 20):
+            q = np.multiply(reconstructed[i_cluster][j], std) + mean
+            reconstructed[i_cluster][j] = q
+
+
+    # ........ infer
+
+    # make a BVH
+    for i_cluster in tqdm(range(0,40)):
+        for j in range(0, 20):
+            filename_prefix = "infered_" +  str(i_cluster) + "_" + str(j)
+            make_bvh(save_path, filename_prefix, reconstructed[i_cluster][j])
+
+
+
+def make_bvh(save_path, filename_prefix, poses):
+    writer = BVHWriter()
+    pipeline = jl.load('../resource/data_pipe.sav')
+
+    # smoothing
+    n_poses = poses.shape[0]
+    out_poses = np.zeros((n_poses, poses.shape[1]))
+
+    for i in range(poses.shape[1]):
+        out_poses[:, i] = savgol_filter(poses[:, i], 15, 2)  # NOTE: smoothing on rotation matrices is not optimal
+
+    # rotation matrix to euler angles
+    out_poses = out_poses.reshape((out_poses.shape[0], -1, 9))
+    out_poses = out_poses.reshape((out_poses.shape[0], out_poses.shape[1], 3, 3))
+    out_euler = np.zeros((out_poses.shape[0], out_poses.shape[1] * 3))
+    for i in range(out_poses.shape[0]):  # frames
+        r = R.from_matrix(out_poses[i])
+        out_euler[i] = r.as_euler('ZXY', degrees=True).flatten()
+
+    bvh_data = pipeline.inverse_transform([out_euler])
+
+    out_bvh_path = os.path.join(save_path, filename_prefix + '_generated.bvh')
+    with open(out_bvh_path, 'w') as f:
+        writer.write(bvh_data[0], f)
+
+
+def plot_loss(checkpoint_path_rnn):
+    x = torch.load(checkpoint_path_rnn)
+    all_eval_loss = x['val_metrics_list']
+    all_train_loss = x['loss_list']
+    # X = np.arange(136-3)
+    # fig = plt.figure()
+    # ax = fig.add_axes([0, 0, 1, 1])
+    # ax.bar(X + 0.00, all_train_loss[0], color='b', width=0.25)
+    # ax.bar(X + 0.25, all_eval_loss[1], color='g', width=0.25)
+    # plt.show()
+    # plotting the second plot
+    plt.plot(all_train_loss, label='Train loss')
+    plt.plot(all_eval_loss, label='Evaluation loss')
+
+    # Labeling the X-axis
+    plt.xlabel('Epoch number')
+    # Labeling the Y-axis
+    plt.ylabel('Loss Average')
+    # Give a title to the graph
+    plt.title('Training/Evaluation Loss based on epoch number')
+
+    # Show a legend on the plot
+    plt.legend()
+
+    # plt.savefig(os.path.join(args.model_save_path, 'loss_plot.png'))
+    plt.show()
+    # exit()
+
+
+if __name__ == '__main__':
+    '''
+
+        ../output/DAE/train_DAE_H41/rep_learning_DAE_H41_checkpoint_020.bin
+        ../output/autoencoder/c2g/autoencode_fxw_zinput_checkpoint_100.bin
+
+    '''
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("ckpt_path_DAE", type=Path)
+    parser.add_argument("ckpt_path_Autoencode", type=Path)
+    args = parser.parse_args()
+
+    # plot_loss(args.ckpt_path_Autoencode)
+    main(args.ckpt_path_DAE, args.ckpt_path_Autoencode)
diff --git a/scripts/inference_text2embedding.py b/scripts/inference_text2embedding.py
new file mode 100644
index 0000000..afdb5fb
--- /dev/null
+++ b/scripts/inference_text2embedding.py
@@ -0,0 +1,774 @@
+import argparse
+import math
+import os
+import pickle
+import pprint
+import random
+from pathlib import Path
+from datetime import datetime
+
+import librosa
+import numpy as np
+import time
+
+import torch
+from scipy.signal import savgol_filter
+import joblib as jl
+
+import utils
+from pymo.preprocessing import *
+from pymo.viz_tools import *
+from pymo.writers import *
+
+
+from utils.data_utils import SubtitleWrapper as SubtitleWrapper_trinity, normalize_string as normalize_string_trinity
+
+from utils.data_utils_twh import SubtitleWrapper as SubtitleWrapper_twh, normalize_string as normalize_string_twh
+
+
+from utils.train_utils import set_logger
+
+from data_loader.data_preprocessor import DataPreprocessor
+
+from inference_Autoencoder import smoothing_function
+
+
+
+from trinity_data_to_lmdb import process_bvh as process_bvh_trinity
+
+from twh_dataset_to_lmdb import process_bvh_test1 as process_bvh_rot_test1
+
+
+
+import matplotlib.pyplot as plt
+import openai
+import os
+import pickle
+import matplotlib.ticker as ticker
+from sklearn.pipeline import Pipeline
+
+
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+infer_from_high_quality = False
+high_quality_model_adress = '../output/autoencoder/toturial/ablation-study/6_Vanilla_DerD/VQ-DVAE_ablation1_checkpoint_020.bin'
+high_quality_model_adress = '../output/IROS_2/AI2_11_HQ/VQVAE_checkpoint_020.bin' # 11-->20f
+high_quality_model_DAE_adress = '../output/IROS_2/DAE_p2/DAE_H40_checkpoint_020.bin'
+
+Inference_audio = True
+openai.api_key = "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"
+
+DATASET_TYPE = 'twh'
+
+
+def GPT_3_caller( str_input, pickle_file_loaded):
+    return 0
+    if pickle_file_loaded != None:
+        for i in range(len(pickle_file_loaded['sample_words_list'])):
+            if str_input == pickle_file_loaded['sample_words_list'][i]:
+                print("Embedding found:", str_input)
+                return pickle_file_loaded['GPT_3_Embedding_list'][i]
+
+    request = openai.Embedding.create(input=str_input, engine="text-similarity-ada-001")
+    embedding = request['data'][0]['embedding']
+    print("!!!!!!!!!!!!!!!!!!!Embedding requested:", str_input)
+    return embedding
+
+def plot_attentions(word_seq, best_guess, attention_list):
+
+    for i in range(len(attention_list)):
+        attention_list[i] = attention_list[i].squeeze().squeeze().cpu().detach().numpy()
+    attention_list = np.array(attention_list)
+
+    best_guess = list(best_guess.squeeze().cpu().detach().numpy())
+    str_best_guess = ''
+    for s in best_guess:
+        str_best_guess += ' ' + str(s)
+
+    # plt.imshow(attention_list)
+    # plt.show()
+    attention_list = attention_list.transpose()
+    # Set up figure with colorbar
+    fig = plt.figure()
+    ax = fig.add_subplot(111)
+    cax = ax.matshow(attention_list, cmap='bone')
+    fig.colorbar(cax)
+
+    # Set up axes
+    ax.set_yticklabels([''] + word_seq.split(' ') +
+                       ['<EOS>'], rotation=0)
+    ax.set_xticklabels([''] + best_guess, rotation=90)
+
+    # Show label at every tick
+    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))
+    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))
+
+    plt.show()
+
+
+def generate_gestures(args, pose_decoder, lang_model,
+                      words, audio_raw, audio_sr, poses,
+                      DAE, Autoencoder, HAutoencoder, GPT3_cach, GPT3_Address):
+
+    # Todo----------------------------------------------
+    # Find nearest neighbour
+
+    # loaded = pickle.load(open('../output/clustering_results/org_latent_clustering_data.bin', 'rb'))
+
+    import os
+    loaded = pickle.load(open(os.path.dirname(args.autoencoder_checkpoint)
+                                       + '/clusters/org_latent_clustering_data.bin', 'rb'))
+
+    kmeanmodel = pickle.load(open(os.path.dirname(args.autoencoder_checkpoint)
+                                       + '/clusters/kmeans_model.pk', 'rb'))
+    # clustering_transforms = pickle.load(open(os.path.dirname(args.autoencoder_checkpoint)
+    #                                               + '/clusters/transforms.pkl', 'rb'))
+
+
+
+    loaded = np.hstack(loaded)
+    # all_dataset_latents = []
+    # count = 0
+    # for item in loaded:
+    #     count = count+1
+    #     if count>2000:
+    #         break
+    #     current_latent_rnn = item['latent_rnn']
+    #     current_latent_rnn = np.hstack(current_latent_rnn)
+    #     all_dataset_latents.append(current_latent_rnn)
+    # all_dataset_latents = np.array(all_dataset_latents)
+    # Todo----------------------------------------------
+
+    # Todo------------------from get sample function-----
+    # loaded  = pickle.load(open('../output/clustering_results/org_latent_clustering_data.bin','rb'))
+    loaded = np.hstack(loaded)
+    # clustering_transforms = pickle.load(open('../output/transforms.pkl', 'rb'))
+
+    if Autoencoder[1].vq: #pose_decoder.decoder.discrete_representation:
+        number_of_clusters = pose_decoder.pose_dim
+        predicted_labels = np.zeros(len(loaded))
+
+        cluster_indexed = dict()
+        for i in range(number_of_clusters):
+            cluster_indexed[i] = []
+
+        for i in range(len(loaded)):
+            current_cluster_index = loaded[i]['quantized_indices'][0]
+            predicted_labels[i] = current_cluster_index
+            cluster_indexed[current_cluster_index].append(i)
+    else:
+        number_of_clusters = 300 # for kmeans
+        data_latent_rnn = []
+        for i in range(len(loaded)):
+            current_latent_rnn = loaded[i]['latent_rnn']
+            current_latent_rnn = np.hstack(current_latent_rnn)
+            data_latent_rnn.append(current_latent_rnn)
+        data_latent_rnn_ndarray = np.array(data_latent_rnn)
+
+        # Applying transformations and clustering.
+        # transfromed = clustering_transforms['scalar'].transform(data_latent_rnn_ndarray)
+        predicted_labels = kmeanmodel.predict(data_latent_rnn_ndarray)
+
+        print("Loop started")
+
+        cluster_indexed = dict()
+        for c_index in range(512):
+            selected_list_indexes = []
+            for ix in range(len(predicted_labels)):
+                if predicted_labels[ix] == c_index:
+                    selected_list_indexes.append(ix)
+                    break # one sample from each cluster
+            cluster_indexed[c_index] = selected_list_indexes
+        print("Loop Done")
+
+    # Todo----------------------------------------------
+
+
+
+    out_list = []
+    clip_length = words[-1][2]
+
+    # pre seq
+    pre_seq = torch.zeros((args.n_pre_poses, pose_decoder.sentence_frame_length // pose_decoder.n_frames))#pose_decoder.pose_dim))
+    pre_seq = torch.zeros(
+        (1, pose_decoder.sentence_frame_length // pose_decoder.n_frames))  # pose_decoder.pose_dim))
+    pre_seq = pre_seq.long()
+    # if seed_seq is not None:
+    #     pre_seq[0, :, :] = torch.Tensor(seed_seq[0:args.n_pre_poses])
+    # else:
+    #     mean_pose = args.data_mean
+    #     mean_pose = torch.squeeze(torch.Tensor(mean_pose))
+    #     pre_seq[0, :, :] = mean_pose.repeat(args.n_pre_poses, 1)
+
+    # divide into inference units and do inferences
+    # unit_time = args.n_poses / args.motion_resampling_framerate
+    unit_time = args.sentence_frame_length / args.motion_resampling_framerate
+    # stride_time = (args.n_poses - args.n_pre_poses) / args.motion_resampling_framerate
+    # Todo: fix this    120
+    stride_time = 4*args.subdivision_stride_sentence / args.motion_resampling_framerate
+
+    if clip_length < unit_time:
+        num_subdivision = 1
+    else:
+        num_subdivision = math.ceil((clip_length - unit_time) / stride_time)
+
+    print('{}, {}, {}, {}'.format(num_subdivision, unit_time, clip_length, stride_time))
+    # num_subdivision = min(num_subdivision, 59)  # DEBUG: generate only for the first N divisions
+    # num_subdivision = min(num_subdivision, 20)  # DEBUG: generate only for the first N divisions
+
+    out_poses = None
+    start = time.time()
+    all_frames_from_rnn = None
+    all_frames_from_original = None
+
+    pre_dim = 45 # DAE[4] #Todo: fix this later
+    decoder_input = torch.zeros((1, pre_dim)).to(device).unsqueeze(0)
+    decoder_input = decoder_input.squeeze(0).unsqueeze(0)
+    decoder_input = decoder_input.transpose(0, 1)
+
+    input_pre_seq = None
+
+    cheat = None
+
+    GPT_3_Embedding_list = []
+    GPT_3_STR_list = []
+    for i in range(0, num_subdivision):
+        # start_time = i * stride_time
+        start_time = i * unit_time
+        end_time = start_time + unit_time
+
+        # Prepare text input
+        word_seq = DataPreprocessor.get_words_in_time_range(word_list=words, start_time=start_time, end_time=end_time)
+        # Todo: check if without sos and eos model performs better.
+        word_indices = np.zeros(len(word_seq)) # + 2)
+        # word_indices[0] = lang_model.SOS_token
+        # word_indices[-1] = lang_model.EOS_token
+        str_input = ""
+        for w_i, word in enumerate(word_seq):
+            print(word[0], end=', ')
+            word_indices[w_i + 0] = lang_model.get_word_index(word[0]) #word[0])
+            str_input += word[0] + ' '
+        str_input = str_input.strip()
+        print(' ({}, {})'.format(start_time, end_time))
+        in_text = torch.LongTensor(word_indices).unsqueeze(0).to(device)
+
+
+        GPT_3_features = GPT_3_caller(str_input, GPT3_cach)
+        GPT_3_features = np.array(GPT_3_features)
+        GPT_3_Embedding_list.append(GPT_3_features)
+        GPT_3_STR_list.append(str_input)
+        # Discretization for the output side:
+        GPT_3_features = torch.from_numpy(GPT_3_features).float().unsqueeze(0).to(device)
+        # Prepare audio
+        if Inference_audio:
+            audio_start = math.floor(start_time * audio_sr)
+            audio_end = int(audio_start + unit_time * audio_sr)
+
+            try:
+                in_audio = (audio_raw[audio_start:audio_end])
+                audi_smaple_rate = 16000
+                mels = []
+                for audio_sub in range(len(in_audio) // audi_smaple_rate):
+                    audio_chunk = in_audio[
+                                  audio_sub * audi_smaple_rate: (audio_sub + 1) * audi_smaple_rate]
+                    signal = librosa.feature.melspectrogram(y=audio_chunk, sr=audi_smaple_rate)
+                    signal = librosa.power_to_db(signal, ref=np.max)
+                    mels.append(signal)
+                    # signal = librosa.amplitude_to_db(signal)
+                in_audio = np.array(mels)
+                in_audio = torch.from_numpy(in_audio).float().to(device)
+                in_audio = in_audio.unsqueeze(0)
+            except:
+                print("xx")
+        else:
+            in_audio = None
+
+        # prepare pre seq
+        # if i > 0:
+        #     pre_seq[0, :, :] = out_poses.squeeze(0)[-args.n_pre_poses:]
+        #     pre_seq[:,0] = out_sentence_latent[].argmax(1)
+        # handled in chet parameter
+        pre_seq = pre_seq.to(device)
+
+        # inference
+        words_lengths = torch.LongTensor([in_text.shape[1]]).to('cpu')
+
+        target_sentence_latent = ""
+
+        print("**********pre_seq", pre_seq)
+
+
+        out_sentence_latent, attention_list = pose_decoder(in_text, words_lengths, in_audio, pre_seq, GPT_3_features, cheat)
+        # plot_attentions(attention_list, word_seq)
+        # out_seq = out_poses[0, :, :].data.cpu().numpy()
+        # todo: wwwcheck:
+        best_guess = out_sentence_latent.argmax(2)
+
+        # Todo: fix it: put best guesses at the begining of pre_seq
+        for i_pre_seq in range(pose_decoder.n_pre_poses):
+            pre_seq[0, i_pre_seq] = best_guess[0, len(best_guess[0])-pose_decoder.n_pre_poses+i_pre_seq]
+
+        cheat = best_guess[:, -1]
+        plot_attentions(str_input, best_guess, attention_list)
+
+
+        # 2. Go through autoencoder
+        Autoencoder_args, Autoencoder_generator, Autoencoder_loss_fn,\
+        Autoencoder_lang_model, Autoencoder_out_dim = Autoencoder
+
+        DAE_args, DAE_generator, DAE_loss_fn, \
+        DAE_lang_model, DAE_out_dim = DAE
+        DAE_generator.train(False)
+
+        # High quality
+        if infer_from_high_quality:
+            HAutoencoder_args, HAutoencoder_generator, HAutoencoder_loss_fn, \
+            HAutoencoder_lang_model, HAutoencoder_out_dim = HAutoencoder
+            HAutoencoder_generator.train(False)
+
+        reconstructed_rnn = torch.zeros(Autoencoder_args.n_poses,
+                                        1,
+                                        Autoencoder_generator.decoder.output_size)\
+            .to(out_sentence_latent.device)
+        out_sentence_latent = out_sentence_latent.squeeze(0)
+
+        # Todo: moved to the top for smoothness
+        # decoder_input = torch.zeros((1, 82)).to(device).unsqueeze(0)
+        # decoder_input = decoder_input.squeeze(0).unsqueeze(0)
+        # decoder_input = decoder_input.transpose(0,1)
+
+
+        for latent_counter in range((out_sentence_latent.size(0))):
+
+            # if Autoencoder_generator.text2_embedding_discrete:
+            if True:
+                decoder_hidden = out_sentence_latent[latent_counter].argmax(0)
+                check1 = out_sentence_latent[latent_counter].to('cpu').detach().numpy()
+                check2 = decoder_hidden.to('cpu').detach().numpy()
+                print("\n--Cluster--> ",check2)
+                original_gestures =\
+                    get_sample_from_dataset(args, loaded=loaded,
+                                            c_index=decoder_hidden,
+                                            cluster_indexed=cluster_indexed,
+                                            seq_length=Autoencoder_args.n_poses)
+
+
+
+                # latent_lin =  latent_lin.to(device)
+                # decoder_hidden = decoder_hidden.to(device).float()
+                # decoder_hidden = decoder_hidden.reshape(Autoencoder_args.n_layers, -1)
+
+
+                target = (original_gestures)
+                target = target.to(device).float()
+                if DAE_generator.encoder == None:
+                    encoded = target
+                else:
+                    encoded = DAE_generator.encoder(target)
+
+                # if torch.eq(encoded , latent_lin):
+                # encoded = latent_lin
+
+                use_derivitive = False
+                if use_derivitive:
+                    diff = [(encoded[n, :] - encoded[n - 1, :]) for n in range(1, encoded.shape[0])]
+                    diff.insert(0, torch.zeros_like(encoded[0, :]))
+                    encoded = torch.hstack((encoded, torch.stack(diff)))
+                input_seq = encoded
+                if input_pre_seq == None:
+                    input_pre_seq = encoded[i] #?
+                input_seq = torch.unsqueeze(input_seq, 0)
+                input_seq = input_seq.transpose(0, 1)
+
+                if infer_from_high_quality:
+                    encoder_outputs, encoder_hidden = HAutoencoder_generator.encoder(input_seq, None)
+                    decoder_hidden = (encoder_hidden[:HAutoencoder_generator.decoder.n_layers])
+                    #Todo: not Variational inference
+                else:
+                    encoder_outputs, encoder_hidden = Autoencoder_generator.encoder(input_seq, None)
+                    decoder_hidden = (encoder_hidden[:Autoencoder_generator.decoder.n_layers])
+
+            else:
+                decoder_hidden = out_sentence_latent[latent_counter]
+                decoder_hidden = decoder_hidden.reshape(Autoencoder_args.n_layers, -1)
+
+            # # Find neaarest neighbour
+            # value = decoder_hidden.clone().cpu().detach().numpy()
+            # idx = np.abs(np.sum(all_dataset_latents - value)).argmin()
+            # decoder_hidden = all_dataset_latents[idx]
+            # decoder_hidden = torch.from_numpy(decoder_hidden).to(device)
+
+
+            decoder_hidden = decoder_hidden.reshape(Autoencoder_args.n_layers, -1)
+            decoder_hidden = decoder_hidden.unsqueeze(1)
+            # decoder_hidden.transpose(0, 1)
+            # output_seq = torch.zeros((30,82))
+            # output_seq = torch.unsqueeze(output_seq, 0)
+            # output_seq = output_seq.transpose(0, 1)
+
+            # decoder_input = torch.zeros_like(decoder_input).to(device)  #moved to the top for smoothness reasons
+
+            for t in range(0, Autoencoder_args.n_poses):
+                # if not Autoencoder_generator.autoencoder_conditioned:
+                #     # Todo: why while it is a conditional network?     This was from autoencoder
+                #     decoder_input = torch.zeros_like(decoder_input).to(device)
+                if infer_from_high_quality:
+                    decoder_output, decoder_hidden, _ = HAutoencoder_generator.decoder(None, decoder_input, decoder_hidden, None,
+                                                                     None)
+                else:
+                    decoder_output, decoder_hidden, _ = Autoencoder_generator.decoder(None, decoder_input,
+                                                                                       decoder_hidden, None,
+                                                                                       None)
+
+                # Todo: fix the smoothness problem: Give the chance of being in the right current
+                # for iq in range(1):
+                #     decoder_output, decoder_hidden, _ = Autoencoder_generator.decoder(None, decoder_input,
+                #                                                                       decoder_hidden, None,
+                #                                                                       None)
+
+
+                try:
+                    reconstructed_rnn[t] = decoder_output
+                except:
+                    print()
+
+                if t < Autoencoder_generator.n_pre_poses and False:
+                    decoder_input = output_seq[t]  # next input is current target
+                else:
+                    decoder_input = decoder_output  # next input is current prediction
+                # Todo: fix this regarding decoder trained autoencoder
+                # if not Autoencoder_generator.autoencoder_conditioned:
+                #     decoder_input = torch.zeros_like(decoder_input)
+
+
+            reconstructed_rnn = encoded.unsqueeze(1)
+
+            if all_frames_from_rnn == None:
+                all_frames_from_rnn = reconstructed_rnn.transpose(0,1)
+            else:
+                all_frames_from_rnn = torch.cat((all_frames_from_rnn, reconstructed_rnn.transpose(0, 1)), 1)
+                # all_frames_from_rnn = torch.cat((all_frames_from_rnn, encoded.transpose(0, 1)), 1)
+
+            if all_frames_from_original == None:
+                all_frames_from_original = target.unsqueeze(0)
+            else:
+                all_frames_from_original = torch.cat((all_frames_from_original, target.unsqueeze(0)), 1)
+
+    #     Todo: decode DAE
+    all_frames_from_rnn = torch.squeeze(all_frames_from_rnn, 0)
+    use_derivitive = Autoencoder_args.use_derivitive == 'True'
+    if use_derivitive:
+        all_frames_from_rnn = all_frames_from_rnn[:, 0: all_frames_from_rnn.shape[1] // 2]
+    if DAE_generator.encoder == None:
+        reconstructed_seq_DAE = all_frames_from_rnn
+    else:
+        reconstructed_seq_DAE = DAE_generator.decoder(all_frames_from_rnn)
+
+    # reconstructed_seq_DAE = torch.squeeze(reconstructed_seq_DAE, 2)
+    reconstructed_seq_DAE = reconstructed_seq_DAE.to('cpu')
+    reconstructed_seq_DAE = reconstructed_seq_DAE.detach().numpy()
+
+    # Todo: just to check sanity
+    # reconstructed_seq_DAE = all_frames_from_original.squeeze().detach().cpu().numpy()
+
+
+
+    print('Avg. inference time: {:.2} s'.format((time.time() - start) / num_subdivision))
+
+    # aggregate results
+
+
+    print("Saving OPENAI GPT3 calss")
+    object_to_save = {'sample_words_list': GPT_3_STR_list,
+                      'GPT_3_Embedding_list': GPT_3_Embedding_list}
+    pickle.dump(object_to_save, open(GPT3_Address, 'wb'))
+
+    return np.array(reconstructed_seq_DAE)
+
+
+def get_sample_from_dataset(args, loaded, c_index, cluster_indexed,  seq_length=30):
+
+    #Todo: fox to use quantization index
+
+    try:
+        selected_list_indexes = cluster_indexed[int(c_index.cpu().detach().numpy())]
+    except:
+        print("22")
+    if selected_list_indexes == []:
+        print("**** Empty list occuered!!!")
+
+    while selected_list_indexes == []:
+        rnd = random.randint(0, len(cluster_indexed) - 1)
+        selected_list_indexes = cluster_indexed[rnd]
+    while True:
+        rnd = random.randint(0, len(selected_list_indexes)-1)
+        if loaded[selected_list_indexes[rnd]]['latent_linear'].shape[0] == seq_length:
+            break
+
+    original = (loaded[selected_list_indexes[rnd]]['original'])
+    print("Chosen:", selected_list_indexes[rnd])
+
+
+    # Since it has been saved as raw data in Clustering.py
+    mean = np.array(args.data_mean).squeeze()
+    std = np.array(args.data_std).squeeze()
+    std = np.clip(std, a_min=0.01, a_max=None)
+    out_poses = ((original) - mean) / std
+    out_poses = torch.from_numpy(out_poses)
+
+
+
+    return out_poses
+
+# def main(checkpoint_path, transcript_path, rep_learning_checkpoint, autoencoder_checkpoint):
+    # GAN = False
+    # if GAN:
+    #     args, generator, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+    #         checkpoint_path, device, what='text2embedding_gan')
+    #     generator = generator.generator
+    # else:
+    #     args, generator, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+    #         checkpoint_path, device, what='text2embedding')
+    #
+    # DAE = utils.train_utils.load_checkpoint_and_model(
+    #     rep_learning_checkpoint, device, what='DAE')
+    # Autoencoder = utils.train_utils.load_checkpoint_and_model(
+    #     autoencoder_checkpoint, device, what='autoencoder_vq')
+    #
+    # HAutoencoder = utils.train_utils.load_checkpoint_and_model(
+    #     high_quality_model_adress, device, what='autoencoder')
+
+def main(checkpoint_path, txt2embedding_model,
+         transcript_path, audio_path, poses_path,
+         DAE, Autoencoder, HAutoencoder):
+
+    args, generator, loss_fn, lang_model, out_dim = txt2embedding_model
+
+    pprint.pprint(vars(args))
+    save_path = '../output/infer_sample'
+    save_path = os.path.dirname(checkpoint_path) + '/infer_sample'
+    os.makedirs(save_path, exist_ok=True)
+
+    # load lang_model
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    with open(vocab_cache_path, 'rb') as f:
+        lang_model = pickle.load(f)
+
+    # prepare input
+    big_infer = False
+    if not big_infer:
+        if DATASET_TYPE == 'trinity':
+            transcript = SubtitleWrapper_trinity(transcript_path).get()
+            # load audio
+            audio_raw, audio_sr = librosa.load((audio_path),
+                                    mono=True, sr=16000, res_type='kaiser_fast')
+            poses, poses_mirror = process_bvh_trinity(poses_path)
+        elif DATASET_TYPE == 'twh':
+            transcript = SubtitleWrapper_twh(transcript_path).get()
+            # load audio
+            audio_raw, audio_sr = librosa.load((audio_path),
+                                               mono=True, sr=16000, res_type='kaiser_fast')
+            poses = process_bvh_rot_test1(poses_path)
+            poses_mirror = poses
+    else:
+        adress = "/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts/TestSeq00"
+        offset = 0
+        transcript_list = []
+        for i in range(1, 5):
+            current_transcript = SubtitleWrapper(adress + str(i) + ".json" ).get()
+            for word in current_transcript:
+                word['start_time'] = "{:.2f}".format(offset + float(word['start_time'][0:-1]) ) + 's'
+                word['end_time'] = "{:.2f}".format(offset + float(word['end_time'][0:-1]) ) + 's'
+            offset = float(current_transcript[-1]['end_time'][0:-1])
+            transcript_list.append(current_transcript)
+        transcript = [item for subist in transcript_list for item in subist]
+
+    GPT3_cach_address = str(transcript_path)[:-5] + '.gpt'
+
+    if os.path.exists(GPT3_cach_address):
+        loaded_GPT = pickle.load(open(GPT3_cach_address,'rb'))
+    else:
+        loaded_GPT = None
+
+    word_list = []
+    for wi in range(len(transcript)):
+        if DATASET_TYPE == 'trinity':
+            word_s = float(transcript[wi]['start_time'][:-1])
+            word_e = float(transcript[wi]['end_time'][:-1])
+            word = transcript[wi]['word']
+
+            word = normalize_string(word)
+            if len(word) > 0:
+                word_list.append([word, word_s, word_e])
+
+        elif DATASET_TYPE == 'twh':
+            for wi in range(len(transcript)):
+                word_s = float(transcript[wi][0])
+                word_e = float(transcript[wi][1])
+                word = transcript[wi][2].strip()
+
+                word_tokens = word.split()
+
+                for t_i, token in enumerate(word_tokens):
+                    token = normalize_string_twh(token)
+                    if len(token) > 0:
+                        new_s_time = word_s + (word_e - word_s) * t_i / len(word_tokens)
+                        new_e_time = word_s + (word_e - word_s) * (t_i + 1) / len(word_tokens)
+                        word_list.append([token, new_s_time, new_e_time])
+
+    # inference
+
+    # Plot motion cluster embedding
+    m_embedding_weights = generator.decoder.decoder.embedding.weight.cpu().detach().numpy()
+    plt.imshow(m_embedding_weights)
+    plt.title("Motion cluster embeddings")
+    plt.show()
+
+
+    out_poses = generate_gestures(args, generator, lang_model,
+                                  word_list, audio_raw, audio_sr, poses,
+                                  DAE, Autoencoder, HAutoencoder,
+                                  loaded_GPT, GPT3_cach_address)
+
+    # unnormalize
+    mean = np.array(args.data_mean).squeeze()
+    std = np.array(args.data_std).squeeze()
+    std = np.clip(std, a_min=0.01, a_max=None)
+    out_poses = np.multiply(out_poses, std) + mean
+
+    # make a BVH
+    filename_prefix = '{}'.format(transcript_path.stem)
+    make_bvh(save_path, filename_prefix, out_poses)
+
+
+def make_bvh(save_path, filename_prefix, poses):
+    writer = BVHWriter()
+    pipeline = jl.load('../resource/data_pipe.sav')
+
+    # smoothing
+    n_poses = poses.shape[0]
+    out_poses = np.zeros((n_poses, poses.shape[1]))
+
+    for i in range(poses.shape[1]):
+        out_poses[:, i] = savgol_filter(poses[:, i], 25, 5)  # NOTE: smoothing on rotation matrices is not optimal
+
+    # rotation matrix to euler angles
+    out_poses = out_poses.reshape((out_poses.shape[0], -1, 9))
+    out_poses = out_poses.reshape((out_poses.shape[0], out_poses.shape[1], 3, 3))
+    out_euler = np.zeros((out_poses.shape[0], out_poses.shape[1] * 3))
+    for i in range(out_poses.shape[0]):  # frames
+        r = R.from_matrix(out_poses[i])
+        out_euler[i] = r.as_euler('ZXY', degrees=True).flatten()
+
+
+    # New smoothing approach in 3D space:
+    out_euler = smoothing_function('spline', out_euler)
+
+
+    bvh_data = pipeline.inverse_transform([out_euler])
+
+    out_bvh_path = os.path.join(save_path, filename_prefix + '.bvh')
+    with open(out_bvh_path, 'w') as f:
+        writer.write(bvh_data[0], f)
+
+
+
+
+
+if __name__ == '__main__':
+
+    '''
+      ../output/autoencoder/toturial/ICLR_text2embedding/VQ-DVAE_ablation1_checkpoint_020.bin
+        /local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts/TestSeq001.json
+        ../output/DAE_old/train_DAE_H41/rep_learning_DAE_H41_checkpoint_020.bin
+        ../output/autoencoder/toturial/4th/VQ-DVAE_ablation1_checkpoint_015.bin
+    
+    
+        ../output/IROS/Proposed4/text2mbedding/VQ-DVAE_ablation1_checkpoint_010.bin
+/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts/TestSeq001.json
+../output/IROS/DAE/DAE_H40_checkpoint_020.bin
+../output/IROS/Proposed4/VQVAE_checkpoint_020.bin
+
+
+../output/GENEA/VQ-VAE/text2mbedding/VQ-DVAE_ablation1_checkpoint_020.bin
+Nothing
+../output/GENEA/DAE/train_DAE_H45/DAE_H45_checkpoint_030.bin
+../output/GENEA/VQ-VAE/VQVAE_checkpoint_3000.bin
+    
+    
+       '''
+    random.seed(datetime.now())
+    parser = argparse.ArgumentParser()
+    parser.add_argument("ckpt_path", type=Path)
+    parser.add_argument("transcript_path", type=Path)
+    parser.add_argument("rep_learning_checkpoint", type=Path)
+    parser.add_argument("autoencoder_checkpoint", type=Path)
+
+    args = parser.parse_args()
+
+    if DATASET_TYPE == 'trinity':
+        pre_transcript_path = "/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts/"
+        pre_audio_path = "/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Audio/"
+        pre_bvh_pth = "/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Motion/"
+    elif DATASET_TYPE == 'twh':
+        pre_path = '/local-scratch/pjomeyaz/rosie_gesture_benchmark/cloned/Clustering/must/GENEA/Co-Speech_Gesture_Generation/dataset/dataset_v1/val'
+
+        pre_transcript_path = pre_path + "/tsv/"
+        pre_audio_path = pre_path + "/wav/"
+        pre_bvh_pth = pre_path + "/bvh/"
+
+
+
+    # From inside the main to increase running time
+    GAN = False
+    if GAN:
+        txt2embedding_model = utils.train_utils.load_checkpoint_and_model(
+            args.ckpt_path, device, what='text2embedding_gan')
+        generator = generator.generator
+    else:
+        txt2embedding_model = utils.train_utils.load_checkpoint_and_model(
+            args.ckpt_path, device, what='text2embedding')
+
+    DAE = utils.train_utils.load_checkpoint_and_model(
+        args.rep_learning_checkpoint, device, what='DAE')
+    Autoencoder = utils.train_utils.load_checkpoint_and_model(
+        args.autoencoder_checkpoint, device, what='autoencoder_vq')
+
+    if infer_from_high_quality:
+        HAutoencoder = utils.train_utils.load_checkpoint_and_model(
+            high_quality_model_adress, device, what='autoencoder_vq')
+        HDAE = utils.train_utils.load_checkpoint_and_model(
+            HAutoencoder[0].rep_learning_checkpoint, device, what='DAE')
+        DAE = HDAE
+    else:
+        HAutoencoder = None
+
+
+    # Generating gestures for each test case:
+
+    if DATASET_TYPE == 'trinity':
+        for i in reversed(range(10, 11)): #11)):
+            transcript_path = pre_transcript_path + "TestSeq" + str(i).zfill(3) + ".json"
+            transcript_path = Path(transcript_path)
+            audio_path = pre_audio_path + "TestSeq" + str(i).zfill(3) + ".wav"
+            poses_path = pre_bvh_pth + "TestSeq" + str(i).zfill(3) + ".bvh"
+            # plot_loss(args.ckpt_path_Autoencode)
+            # main(args.ckpt_path, args.transcript_path, args.rep_learning_checkpoint, args.autoencoder_checkpoint)
+            # main(args.ckpt_path, transcript_path, args.rep_learning_checkpoint, args.autoencoder_checkpoint)
+            main(args.ckpt_path, txt2embedding_model,
+                 transcript_path, audio_path, poses_path,
+                 DAE, Autoencoder, HAutoencoder)
+    elif DATASET_TYPE == 'twh':
+        for i in reversed(range(1, 3)):  # 11)):
+
+            transcript_path = Path(pre_transcript_path + "val_2022_v1_" + str(i).zfill(3) + ".tsv")
+            audio_path = pre_audio_path + "val_2022_v1_" + str(i).zfill(3) + ".wav"
+            poses_path = pre_bvh_pth + "val_2022_v1_" + str(i).zfill(3) + ".bvh"
+
+            # plot_loss(args.ckpt_path_Autoencode)
+            # main(args.ckpt_path, args.transcript_path, args.rep_learning_checkpoint, args.autoencoder_checkpoint)
+            # main(args.ckpt_path, transcript_path, args.rep_learning_checkpoint, args.autoencoder_checkpoint)
+            main(args.ckpt_path, txt2embedding_model,
+                 transcript_path, audio_path, poses_path,
+                 DAE, Autoencoder, HAutoencoder)
diff --git a/scripts/inference_text2embedding_GENEA.py b/scripts/inference_text2embedding_GENEA.py
new file mode 100644
index 0000000..61f4421
--- /dev/null
+++ b/scripts/inference_text2embedding_GENEA.py
@@ -0,0 +1,690 @@
+import argparse
+import math
+import os
+import pickle
+import pprint
+import random
+from pathlib import Path
+from datetime import datetime
+
+import librosa
+import numpy as np
+import time
+
+import torch
+from scipy.signal import savgol_filter
+import joblib as jl
+
+import utils
+from pymo.preprocessing import *
+from pymo.viz_tools import *
+from pymo.writers import *
+from utils.data_utils import SubtitleWrapper, normalize_string
+from utils.train_utils import set_logger
+
+from data_loader.data_preprocessor import DataPreprocessor
+
+from inference_Autoencoder import smoothing_function
+
+from trinity_data_to_lmdb import process_bvh as process_bvh_trinity
+from twh_dataset_to_lmdb import process_bvh as process_bvh_twh
+from twh_dataset_to_lmdb import process_bvh_rot_only as process_bvh_rot_only_twh
+from twh_dataset_to_lmdb import process_bvh_rot_only_Taras as process_bvh_rot_only_Taras
+from inference_DAE import feat2bvh
+
+
+
+import matplotlib.pyplot as plt
+import openai
+import os
+import pickle
+
+
+
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+DATASET_TYPE = 'TWH'
+
+
+infer_from_high_quality = False
+high_quality_model_adress = '../output/autoencoder/toturial/ablation-study/6_Vanilla_DerD/VQ-DVAE_ablation1_checkpoint_020.bin'
+high_quality_model_adress = '../output/IROS_2/AI2_11_HQ/VQVAE_checkpoint_020.bin' # 11-->20f
+high_quality_model_adress = '../output/GENEA/DAE/train_DAE_H20/DAE_H20_checkpoint_050.bin'
+high_quality_model_DAE_adress = '../output/IROS_2/DAE_p2/DAE_H40_checkpoint_020.bin'
+
+Inference_audio = False
+openai.api_key = "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"
+
+def GPT_3_caller( str_input, pickle_file_loaded):
+    if pickle_file_loaded != None:
+        for i in range(len(pickle_file_loaded['sample_words_list'])):
+            if str_input == pickle_file_loaded['sample_words_list'][i]:
+                print("Embedding found:", str_input)
+                return pickle_file_loaded['GPT_3_Embedding_list'][i]
+
+    request = openai.Embedding.create(input=str_input, engine="text-similarity-ada-001")
+    embedding = request['data'][0]['embedding']
+    print("!!!!!!!!!!!!!!!!!!!Embedding requested:", str_input)
+    return embedding
+
+def plot_attentions(attention_list):
+    try:
+        for i in range(len(attention_list)):
+            attention_list[i] = attention_list[i].squeeze().squeeze().cpu().detach().numpy()
+        attention_list = np.array(attention_list)
+        plt.imshow(attention_list)
+        plt.show()
+    except:
+        return
+
+
+def generate_gestures(args, pose_decoder, lang_model,
+                      words, audio_raw, audio_sr, poses,
+                      DAE, Autoencoder, HAutoencoder, GPT3_cach, GPT3_Address):
+
+    # Todo----------------------------------------------
+    # Find nearest neighbour
+
+    # loaded = pickle.load(open('../output/clustering_results/org_latent_clustering_data.bin', 'rb'))
+
+    import os
+    loaded = pickle.load(open(os.path.dirname(args.autoencoder_checkpoint)
+                                       + '/clusters/org_latent_clustering_data.bin', 'rb'))
+
+    kmeanmodel = pickle.load(open(os.path.dirname(args.autoencoder_checkpoint)
+                                       + '/clusters/kmeans_model.pk', 'rb'))
+    # clustering_transforms = pickle.load(open(os.path.dirname(args.autoencoder_checkpoint)
+    #                                               + '/clusters/transforms.pkl', 'rb'))
+
+
+
+    loaded = np.hstack(loaded)
+    # all_dataset_latents = []
+    # count = 0
+    # for item in loaded:
+    #     count = count+1
+    #     if count>2000:
+    #         break
+    #     current_latent_rnn = item['latent_rnn']
+    #     current_latent_rnn = np.hstack(current_latent_rnn)
+    #     all_dataset_latents.append(current_latent_rnn)
+    # all_dataset_latents = np.array(all_dataset_latents)
+    # Todo----------------------------------------------
+
+    # Todo------------------from get sample function-----
+    # loaded  = pickle.load(open('../output/clustering_results/org_latent_clustering_data.bin','rb'))
+    loaded = np.hstack(loaded)
+    # clustering_transforms = pickle.load(open('../output/transforms.pkl', 'rb'))
+
+    if Autoencoder[1].vq: #pose_decoder.decoder.discrete_representation:
+        number_of_clusters = pose_decoder.pose_dim
+        predicted_labels = np.zeros(len(loaded))
+
+        cluster_indexed = dict()
+        for i in range(number_of_clusters):
+            cluster_indexed[i] = []
+
+        for i in range(len(loaded)):
+            current_cluster_index = loaded[i]['quantized_indices'][0]
+            predicted_labels[i] = current_cluster_index
+            cluster_indexed[current_cluster_index].append(i)
+    else:
+        number_of_clusters = 300 # for kmeans
+        data_latent_rnn = []
+        for i in range(len(loaded)):
+            current_latent_rnn = loaded[i]['latent_rnn']
+            current_latent_rnn = np.hstack(current_latent_rnn)
+            data_latent_rnn.append(current_latent_rnn)
+        data_latent_rnn_ndarray = np.array(data_latent_rnn)
+
+        # Applying transformations and clustering.
+        # transfromed = clustering_transforms['scalar'].transform(data_latent_rnn_ndarray)
+        predicted_labels = kmeanmodel.predict(data_latent_rnn_ndarray)
+
+        print("Loop started")
+
+        cluster_indexed = dict()
+        for c_index in range(512):
+            selected_list_indexes = []
+            for ix in range(len(predicted_labels)):
+                if predicted_labels[ix] == c_index:
+                    selected_list_indexes.append(ix)
+                    break # one sample from each cluster
+            cluster_indexed[c_index] = selected_list_indexes
+        print("Loop Done")
+
+    # Todo----------------------------------------------
+
+
+
+    out_list = []
+    clip_length = words[-1][2]
+
+    # pre seq
+    pre_seq = torch.zeros((args.n_pre_poses, pose_decoder.sentence_frame_length // pose_decoder.n_frames))#pose_decoder.pose_dim))
+    pre_seq = torch.zeros(
+        (1, pose_decoder.sentence_frame_length // pose_decoder.n_frames))  # pose_decoder.pose_dim))
+    pre_seq = pre_seq.long()
+    # if seed_seq is not None:
+    #     pre_seq[0, :, :] = torch.Tensor(seed_seq[0:args.n_pre_poses])
+    # else:
+    #     mean_pose = args.data_mean
+    #     mean_pose = torch.squeeze(torch.Tensor(mean_pose))
+    #     pre_seq[0, :, :] = mean_pose.repeat(args.n_pre_poses, 1)
+
+    # divide into inference units and do inferences
+    # unit_time = args.n_poses / args.motion_resampling_framerate
+    unit_time = args.sentence_frame_length / args.motion_resampling_framerate
+    # stride_time = (args.n_poses - args.n_pre_poses) / args.motion_resampling_framerate
+    # Todo: fix this    120
+    stride_time = 4*args.subdivision_stride_sentence / args.motion_resampling_framerate
+
+    if clip_length < unit_time:
+        num_subdivision = 1
+    else:
+        num_subdivision = math.ceil((clip_length - unit_time) / stride_time)
+
+    print('{}, {}, {}, {}'.format(num_subdivision, unit_time, clip_length, stride_time))
+    # num_subdivision = min(num_subdivision, 59)  # DEBUG: generate only for the first N divisions
+    # num_subdivision = min(num_subdivision, 20)  # DEBUG: generate only for the first N divisions
+
+    out_poses = None
+    start = time.time()
+    all_frames_from_rnn = None
+    all_frames_from_original = None
+
+    pre_dim = 20 # DAE[4] #Todo: fix this later
+    decoder_input = torch.zeros((1, pre_dim)).to(device).unsqueeze(0)
+    decoder_input = decoder_input.squeeze(0).unsqueeze(0)
+    decoder_input = decoder_input.transpose(0, 1)
+
+    input_pre_seq = None
+
+    cheat = None
+
+    GPT_3_Embedding_list = []
+    GPT_3_STR_list = []
+    for i in range(0, num_subdivision):
+        # start_time = i * stride_time
+        start_time = i * unit_time
+        end_time = start_time + unit_time
+
+        # Prepare text input
+        word_seq = DataPreprocessor.get_words_in_time_range(word_list=words, start_time=start_time, end_time=end_time)
+        word_indices = np.zeros(len(word_seq) + 2)
+        word_indices[0] = lang_model.SOS_token
+        word_indices[-1] = lang_model.EOS_token
+        str_input = ""
+        for w_i, word in enumerate(word_seq):
+            print(word[0], end=', ')
+            word_indices[w_i + 1] = lang_model.get_word_index(word[0]) #word[0])
+            str_input += word[0] + ' '
+        print(' ({}, {})'.format(start_time, end_time))
+        in_text = torch.LongTensor(word_indices).unsqueeze(0).to(device)
+
+
+        GPT_3_features = GPT_3_caller(str_input, GPT3_cach)
+        GPT_3_features = np.array(GPT_3_features)
+        GPT_3_Embedding_list.append(GPT_3_features)
+        GPT_3_STR_list.append(str_input)
+        # Discretization for the output side:
+        GPT_3_features = torch.from_numpy(GPT_3_features).float().unsqueeze(0).to(device)
+        # Prepare audio
+        if Inference_audio:
+            audio_start = math.floor(start_time * audio_sr)
+            audio_end = int(audio_start + unit_time * audio_sr)
+
+            try:
+                in_audio = (audio_raw[audio_start:audio_end])
+                audi_smaple_rate = 16000
+                mels = []
+                for audio_sub in range(len(in_audio) // audi_smaple_rate):
+                    audio_chunk = in_audio[
+                                  audio_sub * audi_smaple_rate: (audio_sub + 1) * audi_smaple_rate]
+                    signal = librosa.feature.melspectrogram(y=audio_chunk, sr=audi_smaple_rate)
+                    signal = librosa.power_to_db(signal, ref=np.max)
+                    mels.append(signal)
+                    # signal = librosa.amplitude_to_db(signal)
+                in_audio = np.array(mels)
+                in_audio = torch.from_numpy(in_audio).float().to(device)
+                in_audio = in_audio.unsqueeze(0)
+            except:
+                print("xx")
+        else:
+            in_audio = None
+
+        # prepare pre seq
+        # if i > 0:
+        #     pre_seq[0, :, :] = out_poses.squeeze(0)[-args.n_pre_poses:]
+        #     pre_seq[:,0] = out_sentence_latent[].argmax(1)
+        # handled in chet parameter
+        pre_seq = pre_seq.to(device)
+
+        # inference
+        words_lengths = torch.LongTensor([in_text.shape[1]]).to('cpu')
+
+        target_sentence_latent = ""
+
+        out_sentence_latent, attention_list = pose_decoder(in_text, words_lengths, in_audio, pre_seq, GPT_3_features, cheat)
+        plot_attentions(attention_list)
+        # out_seq = out_poses[0, :, :].data.cpu().numpy()
+        # todo: wwwcheck:
+        best_guess = out_sentence_latent.argmax(2)
+        cheat = best_guess[:, -1]
+
+
+        # 2. Go through autoencoder
+        Autoencoder_args, Autoencoder_generator, Autoencoder_loss_fn,\
+        Autoencoder_lang_model, Autoencoder_out_dim = Autoencoder
+
+        DAE_args, DAE_generator, DAE_loss_fn, \
+        DAE_lang_model, DAE_out_dim = DAE
+        DAE_generator.train(False)
+
+        # High quality
+        if infer_from_high_quality:
+            HAutoencoder_args, HAutoencoder_generator, HAutoencoder_loss_fn, \
+            HAutoencoder_lang_model, HAutoencoder_out_dim = HAutoencoder
+            HAutoencoder_generator.train(False)
+        reconstructed_rnn = torch.zeros(Autoencoder_args.n_poses,
+                                        1,
+                                        Autoencoder_generator.decoder.output_size)\
+            .to(out_sentence_latent.device)
+        out_sentence_latent = out_sentence_latent.squeeze(0)
+
+        # Todo: moved to the top for smoothness
+        # decoder_input = torch.zeros((1, 82)).to(device).unsqueeze(0)
+        # decoder_input = decoder_input.squeeze(0).unsqueeze(0)
+        # decoder_input = decoder_input.transpose(0,1)
+
+
+        for latent_counter in range((out_sentence_latent.size(0))):
+
+            # if Autoencoder_generator.text2_embedding_discrete:
+            if True:
+                decoder_hidden = out_sentence_latent[latent_counter].argmax(0)
+                check1 = out_sentence_latent[latent_counter].to('cpu').detach().numpy()
+                check2 = decoder_hidden.to('cpu').detach().numpy()
+                print("\n--Cluster--> ",check2)
+                original_gestures =\
+                    get_sample_from_dataset(args, loaded=loaded,
+                                            c_index=decoder_hidden,
+                                            cluster_indexed=cluster_indexed,
+                                            seq_length=Autoencoder_args.n_poses)
+
+
+
+                # latent_lin =  latent_lin.to(device)
+                # decoder_hidden = decoder_hidden.to(device).float()
+                # decoder_hidden = decoder_hidden.reshape(Autoencoder_args.n_layers, -1)
+
+
+                target = (original_gestures)
+                target = target.to(device).float()
+                if DAE_generator.encoder == None:
+                    encoded = target
+                else:
+                    encoded = DAE_generator.encoder(target)
+
+                # if torch.eq(encoded , latent_lin):
+                # encoded = latent_lin
+
+                use_derivitive = False
+                if use_derivitive:
+                    diff = [(encoded[n, :] - encoded[n - 1, :]) for n in range(1, encoded.shape[0])]
+                    diff.insert(0, torch.zeros_like(encoded[0, :]))
+                    encoded = torch.hstack((encoded, torch.stack(diff)))
+                input_seq = encoded
+                if input_pre_seq == None:
+                    input_pre_seq = encoded[i] #?
+                input_seq = torch.unsqueeze(input_seq, 0)
+                input_seq = input_seq.transpose(0, 1)
+
+                if infer_from_high_quality:
+                    encoder_outputs, encoder_hidden = HAutoencoder_generator.encoder(input_seq, None)
+                    decoder_hidden = (encoder_hidden[:HAutoencoder_generator.decoder.n_layers])
+                    #Todo: not Variational inference
+                else:
+                    encoder_outputs, encoder_hidden = Autoencoder_generator.encoder(input_seq, None)
+                    decoder_hidden = (encoder_hidden[:Autoencoder_generator.decoder.n_layers])
+
+            else:
+                decoder_hidden = out_sentence_latent[latent_counter]
+                decoder_hidden = decoder_hidden.reshape(Autoencoder_args.n_layers, -1)
+
+            # # Find neaarest neighbour
+            # value = decoder_hidden.clone().cpu().detach().numpy()
+            # idx = np.abs(np.sum(all_dataset_latents - value)).argmin()
+            # decoder_hidden = all_dataset_latents[idx]
+            # decoder_hidden = torch.from_numpy(decoder_hidden).to(device)
+
+
+            decoder_hidden = decoder_hidden.reshape(Autoencoder_args.n_layers, -1)
+            decoder_hidden = decoder_hidden.unsqueeze(1)
+            # decoder_hidden.transpose(0, 1)
+            # output_seq = torch.zeros((30,82))
+            # output_seq = torch.unsqueeze(output_seq, 0)
+            # output_seq = output_seq.transpose(0, 1)
+
+            # decoder_input = torch.zeros_like(decoder_input).to(device)  #moved to the top for smoothness reasons
+
+            for t in range(0, Autoencoder_args.n_poses):
+                # if not Autoencoder_generator.autoencoder_conditioned:
+                #     # Todo: why while it is a conditional network?     This was from autoencoder
+                #     decoder_input = torch.zeros_like(decoder_input).to(device)
+                if infer_from_high_quality:
+                    decoder_output, decoder_hidden, _ = HAutoencoder_generator.decoder(None, decoder_input, decoder_hidden, None,
+                                                                     None)
+                else:
+                    decoder_output, decoder_hidden, _ = Autoencoder_generator.decoder(None, decoder_input,
+                                                                                       decoder_hidden, None,
+                                                                                       None)
+
+                # Todo: fix the smoothness problem: Give the chance of being in the right current
+                # for iq in range(1):
+                #     decoder_output, decoder_hidden, _ = Autoencoder_generator.decoder(None, decoder_input,
+                #                                                                       decoder_hidden, None,
+                #                                                                       None)
+
+
+                try:
+                    reconstructed_rnn[t] = decoder_output
+                except:
+                    print()
+
+                if t < Autoencoder_generator.n_pre_poses and False:
+                    decoder_input = output_seq[t]  # next input is current target
+                else:
+                    decoder_input = decoder_output  # next input is current prediction
+                # Todo: fix this regarding decoder trained autoencoder
+                # if not Autoencoder_generator.autoencoder_conditioned:
+                #     decoder_input = torch.zeros_like(decoder_input)
+
+
+            reconstructed_rnn = encoded.unsqueeze(1)
+
+            if all_frames_from_rnn == None:
+                all_frames_from_rnn = reconstructed_rnn.transpose(0,1)
+            else:
+                all_frames_from_rnn = torch.cat((all_frames_from_rnn, reconstructed_rnn.transpose(0, 1)), 1)
+                # all_frames_from_rnn = torch.cat((all_frames_from_rnn, encoded.transpose(0, 1)), 1)
+
+            if all_frames_from_original == None:
+                all_frames_from_original = target.unsqueeze(0)
+            else:
+                all_frames_from_original = torch.cat((all_frames_from_original, target.unsqueeze(0)), 1)
+
+    #     Todo: decode DAE
+    all_frames_from_rnn = torch.squeeze(all_frames_from_rnn, 0)
+    if infer_from_high_quality:
+        use_derivitive = HAutoencoder_args.use_derivitive == 'True'
+    else:
+        use_derivitive = Autoencoder_args.use_derivitive == 'True'
+    if use_derivitive:
+        all_frames_from_rnn = all_frames_from_rnn[:, 0: all_frames_from_rnn.shape[1] // 2]
+    if DAE_generator.encoder == None:
+        reconstructed_seq_DAE = all_frames_from_rnn
+    else:
+        reconstructed_seq_DAE = DAE_generator.decoder(all_frames_from_rnn)
+
+    # reconstructed_seq_DAE = torch.squeeze(reconstructed_seq_DAE, 2)
+    reconstructed_seq_DAE = reconstructed_seq_DAE.to('cpu')
+    reconstructed_seq_DAE = reconstructed_seq_DAE.detach().numpy()
+
+    # Todo: just to check sanity
+    # reconstructed_seq_DAE = all_frames_from_original.squeeze().detach().cpu().numpy()
+
+
+
+    print('Avg. inference time: {:.2} s'.format((time.time() - start) / num_subdivision))
+
+    # aggregate results
+
+
+    print("Saving OPENAI GPT3 calss")
+    object_to_save = {'sample_words_list': GPT_3_STR_list,
+                      'GPT_3_Embedding_list': GPT_3_Embedding_list}
+    pickle.dump(object_to_save, open(GPT3_Address, 'wb'))
+
+    return np.array(reconstructed_seq_DAE)
+
+
+def get_sample_from_dataset(args, loaded, c_index, cluster_indexed,  seq_length=30):
+
+    #Todo: fox to use quantization index
+
+    try:
+        selected_list_indexes = cluster_indexed[int(c_index.cpu().detach().numpy())]
+    except:
+        print("22")
+    if selected_list_indexes == []:
+        print("**** Empty list occuered!!!")
+
+    while selected_list_indexes == []:
+        rnd = random.randint(0, len(cluster_indexed) - 1)
+        selected_list_indexes = cluster_indexed[rnd]
+    while True:
+        rnd = random.randint(0, len(selected_list_indexes)-1)
+        if loaded[selected_list_indexes[rnd]]['latent_linear'].shape[0] == seq_length:
+            break
+
+    original = (loaded[selected_list_indexes[rnd]]['original'])
+    print("Chosen:", selected_list_indexes[rnd])
+
+
+    # Since it has been saved as raw data in Clustering.py
+    mean = np.array(args.data_mean).squeeze()
+    std = np.array(args.data_std).squeeze()
+    std = np.clip(std, a_min=0.01, a_max=None)
+    out_poses = ((original) - mean) / std
+    out_poses = torch.from_numpy(out_poses)
+
+
+
+    return out_poses
+
+# def main(checkpoint_path, transcript_path, rep_learning_checkpoint, autoencoder_checkpoint):
+    # GAN = False
+    # if GAN:
+    #     args, generator, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+    #         checkpoint_path, device, what='text2embedding_gan')
+    #     generator = generator.generator
+    # else:
+    #     args, generator, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+    #         checkpoint_path, device, what='text2embedding')
+    #
+    # DAE = utils.train_utils.load_checkpoint_and_model(
+    #     rep_learning_checkpoint, device, what='DAE')
+    # Autoencoder = utils.train_utils.load_checkpoint_and_model(
+    #     autoencoder_checkpoint, device, what='autoencoder_vq')
+    #
+    # HAutoencoder = utils.train_utils.load_checkpoint_and_model(
+    #     high_quality_model_adress, device, what='autoencoder')
+
+def main(checkpoint_path, txt2embedding_model,
+         transcript_path, audio_path, poses_path,
+         DAE, Autoencoder, HAutoencoder):
+
+    args, generator, loss_fn, lang_model, out_dim = txt2embedding_model
+
+    pprint.pprint(vars(args))
+    save_path = '../output/infer_sample'
+    save_path = os.path.dirname(checkpoint_path) + '/infer_sample'
+    os.makedirs(save_path, exist_ok=True)
+
+    # load lang_model
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    with open(vocab_cache_path, 'rb') as f:
+        lang_model = pickle.load(f)
+
+    # prepare input
+    big_infer = False
+    if not big_infer:
+        transcript = SubtitleWrapper(transcript_path).get()
+        # load audio
+        audio_raw, audio_sr = librosa.load((audio_path),
+                                mono=True, sr=16000, res_type='kaiser_fast')
+
+        if DATASET_TYPE == 'Trinity':
+            poses, poses_mirror = process_bvh_trinity(poses_path)
+        elif DATASET_TYPE == 'TWH':
+            poses = process_bvh_rot_only_Taras(poses_path)
+            poses_mirror = poses
+
+    else:
+        adress = "/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts/TestSeq00"
+        offset = 0
+        transcript_list = []
+        for i in range(1, 5):
+            current_transcript = SubtitleWrapper(adress + str(i) + ".json" ).get()
+            for word in current_transcript:
+                word['start_time'] = "{:.2f}".format(offset + float(word['start_time'][0:-1]) ) + 's'
+                word['end_time'] = "{:.2f}".format(offset + float(word['end_time'][0:-1]) ) + 's'
+            offset = float(current_transcript[-1]['end_time'][0:-1])
+            transcript_list.append(current_transcript)
+        transcript = [item for subist in transcript_list for item in subist]
+
+    GPT3_cach_address = str(transcript_path)[:-5] + '.gpt'
+
+    if os.path.exists(GPT3_cach_address):
+        loaded_GPT = pickle.load(open(GPT3_cach_address,'rb'))
+    else:
+        loaded_GPT = None
+
+    word_list = []
+    for wi in range(len(transcript)):
+        word_s = float(transcript[wi]['start_time'][:-1])
+        word_e = float(transcript[wi]['end_time'][:-1])
+        word = transcript[wi]['word']
+
+        word = normalize_string(word)
+        if len(word) > 0:
+            word_list.append([word, word_s, word_e])
+
+    # inference
+
+    # Plot motion cluster embedding
+    m_embedding_weights = generator.decoder.decoder.embedding.weight.cpu().detach().numpy()
+    plt.imshow(m_embedding_weights)
+    plt.title("Motion cluster embeddings")
+    plt.show()
+
+
+    out_poses = generate_gestures(args, generator, lang_model,
+                                  word_list, audio_raw, audio_sr, poses,
+                                  DAE, Autoencoder, HAutoencoder,
+                                  loaded_GPT, GPT3_cach_address)
+
+    # unnormalize
+    mean = np.array(args.data_mean).squeeze()
+    std = np.array(args.data_std).squeeze()
+    std = np.clip(std, a_min=0.01, a_max=None)
+    out_poses = np.multiply(out_poses, std) + mean
+
+    # make a BVH
+    filename_prefix = '{}'.format(transcript_path.stem)
+    if DATASET_TYPE=='Trinity':
+        make_bvh(save_path, filename_prefix, out_poses)
+    elif DATASET_TYPE == 'TWH':
+        feat2bvh(save_path, filename_prefix, out_poses)
+
+def make_bvh(save_path, filename_prefix, poses):
+    writer = BVHWriter()
+    pipeline = jl.load('../resource/data_pipe.sav')
+
+    # smoothing
+    n_poses = poses.shape[0]
+    out_poses = np.zeros((n_poses, poses.shape[1]))
+
+    for i in range(poses.shape[1]):
+        out_poses[:, i] = savgol_filter(poses[:, i], 25, 5)  # NOTE: smoothing on rotation matrices is not optimal
+
+    # rotation matrix to euler angles
+    out_poses = out_poses.reshape((out_poses.shape[0], -1, 9))
+    out_poses = out_poses.reshape((out_poses.shape[0], out_poses.shape[1], 3, 3))
+    out_euler = np.zeros((out_poses.shape[0], out_poses.shape[1] * 3))
+    for i in range(out_poses.shape[0]):  # frames
+        r = R.from_matrix(out_poses[i])
+        out_euler[i] = r.as_euler('ZXY', degrees=True).flatten()
+
+
+    # New smoothing approach in 3D space:
+    out_euler = smoothing_function('spline', out_euler)
+
+
+    bvh_data = pipeline.inverse_transform([out_euler])
+
+    out_bvh_path = os.path.join(save_path, filename_prefix + '.bvh')
+    with open(out_bvh_path, 'w') as f:
+        writer.write(bvh_data[0], f)
+
+
+if __name__ == '__main__':
+
+    '''
+      ../output/autoencoder/toturial/ICLR_text2embedding/VQ-DVAE_ablation1_checkpoint_020.bin
+        /local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts/TestSeq001.json
+        ../output/DAE_old/train_DAE_H41/rep_learning_DAE_H41_checkpoint_020.bin
+        ../output/autoencoder/toturial/4th/VQ-DVAE_ablation1_checkpoint_015.bin
+    
+    
+        ../output/IROS/Proposed4/text2mbedding/VQ-DVAE_ablation1_checkpoint_010.bin
+/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts/TestSeq001.json
+../output/IROS/DAE/DAE_H40_checkpoint_020.bin
+../output/IROS/Proposed4/VQVAE_checkpoint_020.bin
+
+
+../output/GENEA/VQ-VAE/text2mbedding/VQ-DVAE_ablation1_checkpoint_050.bin
+Nothing
+../output/GENEA/DAE/train_DAE_H20/DAE_H20_checkpoint_050.bin
+../output/GENEA/VQ-VAE/VQVAE_checkpoint_020.bin
+    
+    
+       '''
+    random.seed(datetime.now())
+    parser = argparse.ArgumentParser()
+    parser.add_argument("ckpt_path", type=Path)
+    parser.add_argument("transcript_path", type=Path)
+    parser.add_argument("rep_learning_checkpoint", type=Path)
+    parser.add_argument("autoencoder_checkpoint", type=Path)
+
+    args = parser.parse_args()
+
+    pre_transcript_path = "/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts/"
+    pre_audio_path = "/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Audio/"
+    pre_bvh_pth = "/local-scratch/pjomeyaz/rosie_gesture_benchmark/cloned/Clustering/must/GENEA/Co-Speech_Gesture_Generation/dataset/dataset_v1/val/bvh/"
+    # From inside the main to increase running time
+    GAN = False
+    if GAN:
+        txt2embedding_model = utils.train_utils.load_checkpoint_and_model(
+            args.ckpt_path, device, what='text2embedding_gan')
+        generator = generator.generator
+    else:
+        txt2embedding_model = utils.train_utils.load_checkpoint_and_model(
+            args.ckpt_path, device, what='text2embedding')
+
+    DAE = utils.train_utils.load_checkpoint_and_model(
+        args.rep_learning_checkpoint, device, what='DAE')
+    Autoencoder = utils.train_utils.load_checkpoint_and_model(
+        args.autoencoder_checkpoint, device, what='autoencoder_vq')
+
+    HAutoencoder = None
+    if infer_from_high_quality:
+        HAutoencoder = utils.train_utils.load_checkpoint_and_model(
+            high_quality_model_adress, device, what='autoencoder_vq')
+        HDAE = utils.train_utils.load_checkpoint_and_model(
+            HAutoencoder[0].rep_learning_checkpoint, device, what='DAE')
+
+        DAE = HDAE
+
+    for i in reversed(range(1, 2)): #11)):
+        transcript_path = pre_transcript_path + "TestSeq" + str(i).zfill(3) + ".json"
+        transcript_path = Path(transcript_path)
+        audio_path = pre_audio_path + "TestSeq" + str(i).zfill(3) + ".wav"
+        poses_path = pre_bvh_pth + "val_2022_v1_" + str(i).zfill(3) + ".bvh"
+        # plot_loss(args.ckpt_path_Autoencode)
+        # main(args.ckpt_path, args.transcript_path, args.rep_learning_checkpoint, args.autoencoder_checkpoint)
+        # main(args.ckpt_path, transcript_path, args.rep_learning_checkpoint, args.autoencoder_checkpoint)
+        main(args.ckpt_path, txt2embedding_model,
+             transcript_path, audio_path, poses_path,
+             DAE, Autoencoder, HAutoencoder)
diff --git a/scripts/model/Autoencoder_VQVAE_model.py b/scripts/model/Autoencoder_VQVAE_model.py
new file mode 100644
index 0000000..f59904c
--- /dev/null
+++ b/scripts/model/Autoencoder_VQVAE_model.py
@@ -0,0 +1,1383 @@
+import torch
+from torch.autograd import Variable
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+
+'''
+Based on the following Se2Seq implementations:
+- https://github.com/AuCson/PyTorch-Batch-Attention-Seq2seq
+- https://github.com/spro/practical-pytorch/blob/master/seq2seq-translation/seq2seq-translation-batched.ipynb
+'''
+
+debug = False
+
+class EncoderRNN(nn.Module):
+    def __init__(self, input_size, hidden_size, n_layers=1, dropout=0.5, pre_trained_embedding=None):
+        super(EncoderRNN, self).__init__()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.n_layers = n_layers
+        self.dropout = dropout
+
+        self.in_layer = nn.Linear(input_size, hidden_size)
+        self.gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=self.dropout, bidirectional=True)
+
+
+        self.do_flatten_parameters = False
+        if torch.cuda.device_count() > 1:
+            self.do_flatten_parameters = True
+
+    def forward(self, input_seqs, hidden=None):
+        '''
+        :param input_seqs:
+            Variable of shape (num_step(T),batch_size(B)), sorted decreasingly by lengths(for packing)
+        :param input_lengths:
+            list of sequence length
+        :param hidden:
+            initial state of GRU
+        :returns:
+            GRU outputs in shape (T,B,hidden_size(H))
+            last hidden stat of RNN(i.e. last output for GRU)
+        '''
+        if self.do_flatten_parameters:
+            self.gru.flatten_parameters()
+        if debug:
+            print("Encoder_Forward_input", input_seqs.shape)
+        input_seq_in_layered = self.in_layer(input_seqs)
+        outputs, hidden = self.gru(input_seq_in_layered, hidden)
+        # outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(outputs)  # unpack (back to padded)
+        outputs = outputs[:, :, :self.hidden_size] + outputs[:, :, self.hidden_size:]  # Sum bidirectional outputs
+        if debug:
+            print("Encoder_Forward_ out, hidden", outputs.shape, hidden.shape)
+        return outputs, hidden
+
+class EncoderCNN(nn.Module):
+    def __init__(self, input_size, hidden_size, n_layers=1, dropout=0.5, pre_trained_embedding=None):
+        super(EncoderCNN, self).__init__()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.n_layers = n_layers
+        self.dropout = dropout
+
+        self.in_layer = nn.Linear(input_size, hidden_size)
+        self.gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=self.dropout, bidirectional=True)
+
+        self.cnn = nn.Conv1d(in_channels=1,out_channels=2,kernel_size=2, stride=2)
+        channels = [hidden_size, hidden_size, hidden_size, hidden_size, hidden_size, hidden_size]
+        self.cnn = nn.Sequential(
+            nn.Conv1d(in_channels=channels[0],out_channels=channels[1],kernel_size=5, stride=5, bias=False),
+            nn.BatchNorm1d(channels[1], affine=True),
+            nn.ReLU(inplace=True),
+            nn.Conv1d(in_channels=channels[1],out_channels=channels[2],kernel_size=3, stride=3, bias=False),
+            nn.BatchNorm1d(channels[2], affine=True),
+            nn.ReLU(inplace=True),
+            nn.Conv1d(in_channels=channels[2],out_channels=channels[3],kernel_size=2, stride=2, bias=False),
+            nn.BatchNorm1d(channels[3], affine=True),
+            nn.ReLU(inplace=True),
+            # nn.Conv1d(in_channels=channels[3],out_channels=channels[4],kernel_size=2, stride=2, bias=False),
+            # nn.BatchNorm1d(channels[4], affine=True),
+            # nn.ReLU(inplace=True),
+            # nn.Conv1d(in_channels=channels[4],out_channels=channels[5],kernel_size=2, stride=2, bias=False),
+        )
+
+
+
+        self.do_flatten_parameters = False
+        if torch.cuda.device_count() > 1:
+            self.do_flatten_parameters = True
+
+    def forward(self, input_seqs, hidden=None):
+        '''
+        :param input_seqs:
+            Variable of shape (num_step(T),batch_size(B)), sorted decreasingly by lengths(for packing)
+        :param input_lengths:
+            list of sequence length
+        :param hidden:
+            initial state of GRU
+        :returns:
+            GRU outputs in shape (T,B,hidden_size(H))
+            last hidden stat of RNN(i.e. last output for GRU)
+        '''
+        if self.do_flatten_parameters:
+            self.gru.flatten_parameters()
+        if debug:
+            print("Encoder_Forward_input", input_seqs.shape)
+        input_seq_in_layered = self.in_layer(input_seqs)
+        input_seq_in_layered = input_seq_in_layered.permute((1,2,0)) #128,30,200 --> 128,200,30 n*c*l
+        # outputs, hidden = self.gru(input_seq_in_layered, hidden)
+
+
+        print(self.cnn)
+        hidden = self.cnn(input_seq_in_layered)
+        # outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(outputs)  # unpack (back to padded)
+        # outputs = outputs[:, :, :self.hidden_size] + outputs[:, :, self.hidden_size:]  # Sum bidirectional outputs
+        outputs = torch.tensor(0)
+        if debug:
+            print("Encoder_Forward_ out, hidden", outputs.shape, hidden.shape)
+        return outputs, hidden
+class DecoderCNN(nn.Module):
+    def __init__(self, input_size, hidden_size, n_layers=1, dropout=0.5, pre_trained_embedding=None):
+        super(DecoderCNN, self).__init__()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.n_layers = n_layers
+        self.dropout = dropout
+
+        self.in_layer = nn.Linear(input_size, hidden_size)
+        self.gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=self.dropout, bidirectional=True)
+        self.cnn = nn.Conv1d(in_channels=1,out_channels=2,kernel_size=2, stride=2)
+        channels = [hidden_size, hidden_size, hidden_size, hidden_size, hidden_size, hidden_size]
+        self.cnn = nn.Sequential(
+            nn.ConvTranspose1d(in_channels=channels[0],out_channels=channels[1],kernel_size=2, stride=2, bias=False),
+            nn.BatchNorm1d(channels[1], affine=True),
+            nn.ReLU(inplace=True),
+
+            nn.ConvTranspose1d(in_channels=channels[1],out_channels=channels[2],kernel_size=3, stride=3, bias=False),
+            nn.BatchNorm1d(channels[2], affine=True),
+            nn.ReLU(inplace=True),
+
+            nn.ConvTranspose1d(in_channels=channels[2],out_channels=channels[3],kernel_size=5, stride=5, bias=False),
+            nn.BatchNorm1d(channels[3], affine=True),
+            nn.ReLU(inplace=True),
+
+            # nn.Conv1d(in_channels=channels[3],out_channels=channels[4],kernel_size=2, stride=2, bias=False),
+            # nn.BatchNorm1d(channels[4], affine=True),
+            # nn.ReLU(inplace=True),
+            # nn.Conv1d(in_channels=channels[4],out_channels=channels[5],kernel_size=2, stride=2, bias=False),
+        )
+
+
+
+        self.do_flatten_parameters = False
+        if torch.cuda.device_count() > 1:
+            self.do_flatten_parameters = True
+
+    def forward(self, input_seqs, hidden=None):
+        '''
+        :param input_seqs:
+            Variable of shape (num_step(T),batch_size(B)), sorted decreasingly by lengths(for packing)
+        :param input_lengths:
+            list of sequence length
+        :param hidden:
+            initial state of GRU
+        :returns:
+            GRU outputs in shape (T,B,hidden_size(H))
+            last hidden stat of RNN(i.e. last output for GRU)
+        '''
+        if self.do_flatten_parameters:
+            self.gru.flatten_parameters()
+        if debug:
+            print("Encoder_Forward_input", input_seqs.shape)
+            print(self.in_layer)
+        input_seq_in_layered = self.in_layer(input_seqs.squeeze()).unsqueeze(-1)
+        input_seq_in_layered = input_seq_in_layered.unsqueeze(0)
+        # input_seq_in_layered = input_seq_in_layered.permute((1,2,0)) #128,30,200 --> 128,200,30 n*c*l
+        # outputs, hidden = self.gru(input_seq_in_layered, hidden)
+
+        print("DCNN input_seq_in_layered", input_seq_in_layered.shape)
+
+        outputs = self.cnn(input_seq_in_layered)
+        # print("DCNN outputs", outputs.shape)
+        # outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(outputs)  # unpack (back to padded)
+        # outputs = outputs[:, :, :self.hidden_size] + outputs[:, :, self.hidden_size:]  # Sum bidirectional outputs
+        hidden = torch.tensor(0)
+        if debug:
+            print("Encoder_Forward_ out, hidden", outputs.shape, hidden.shape)
+        return outputs, hidden
+
+class Attn(nn.Module):
+    def __init__(self, hidden_size):
+        super(Attn, self).__init__()
+        self.hidden_size = hidden_size
+        self.attn = nn.Linear(self.hidden_size * 2, hidden_size)
+        self.v = nn.Parameter(torch.rand(hidden_size))
+        stdv = 1. / math.sqrt(self.v.size(0))
+        self.v.data.normal_(mean=0, std=stdv)
+
+    def forward(self, hidden, encoder_outputs):
+        '''
+        :param hidden:
+            previous hidden state of the decoder, in shape (layers*directions,B,H)
+        :param encoder_outputs:
+            encoder outputs from Encoder, in shape (T,B,H)
+        :return
+            attention energies in shape (B,T)
+        '''
+        max_len = encoder_outputs.size(0)
+        this_batch_size = encoder_outputs.size(1)
+        H = hidden.repeat(max_len, 1, 1).transpose(0, 1)
+        encoder_outputs = encoder_outputs.transpose(0, 1)  # [B*T*H]
+        attn_energies = self.score(H, encoder_outputs)  # compute attention score
+        return F.softmax(attn_energies, dim=1).unsqueeze(1)  # normalize with softmax
+
+    def score(self, hidden, encoder_outputs):
+        energy = torch.tanh(self.attn(torch.cat([hidden, encoder_outputs], 2)))  # [B*T*2H]->[B*T*H]
+        energy = energy.transpose(2, 1)  # [B*H*T]
+        v = self.v.repeat(encoder_outputs.data.shape[0], 1).unsqueeze(1)  # [B*1*H]
+        energy = torch.bmm(v, energy)  # [B*1*T]
+        return energy.squeeze(1)  # [B*T]
+
+
+class BahdanauAttnDecoderRNN(nn.Module):
+    def __init__(self, args, input_size, hidden_size, output_size, n_layers=1, dropout_p=0.1,
+                 discrete_representation=False, speaker_model=None):
+        super(BahdanauAttnDecoderRNN, self).__init__()
+
+        # define parameters
+        self.hidden_size = hidden_size
+        self.output_size = output_size
+        self.n_layers = n_layers
+        self.dropout_p = dropout_p
+        self.discrete_representation = discrete_representation
+        self.speaker_model = speaker_model
+
+        # define embedding layer
+        if self.discrete_representation:
+            self.embedding = nn.Embedding(output_size, hidden_size)
+            self.dropout = nn.Dropout(dropout_p)
+
+        # if self.speaker_model:
+        #     self.speaker_embedding = nn.Embedding(speaker_model.n_words, 8)
+
+        # calc input size
+        # if self.discrete_representation:
+        #     input_size = hidden_size  # embedding size
+        # linear_input_size = input_size + hidden_size
+        # if self.speaker_model:
+        #     linear_input_size += 8
+
+        if args.autoencoder_conditioned == 'True':
+            self.autoencoder_conditioned = True
+        else:
+            self.autoencoder_conditioned = False
+
+        # define layers
+
+        if args.autoencoder_att == 'True':
+            self.attn = Attn(hidden_size)
+            self.att_use = True
+        else:
+            self.att_use = False
+
+        if self.att_use:
+            linear_input_size = input_size + hidden_size
+        else:
+            linear_input_size = input_size
+        self.pre_linear = nn.Sequential(
+            nn.Linear(linear_input_size, hidden_size),
+            nn.BatchNorm1d(hidden_size),
+            nn.ReLU(inplace=True)
+        )
+
+
+        self.gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=dropout_p)
+        if args.autoencoder_fixed_weight == 'True':
+            self.autoencoder_fixed_weight = True
+            for param in self.gru.parameters():
+                param.requires_grad = False
+
+
+
+        # self.out = nn.Linear(hidden_size * 2, output_size)
+        self.out_layer = nn.Linear(hidden_size, output_size)
+
+        self.do_flatten_parameters = False
+        if torch.cuda.device_count() > 1:
+            self.do_flatten_parameters = True
+
+    def freeze_attn(self):
+        for param in self.attn.parameters():
+            param.requires_grad = False
+
+    def forward(self, motion_input, last_hidden, encoder_outputs, vid_indices=None):
+        '''
+        :param motion_input:
+            motion input for current time step, in shape [batch x dim]
+        :param last_hidden:
+            last hidden state of the decoder, in shape [layers x batch x hidden_size]
+        :param encoder_outputs:
+            encoder outputs in shape [steps x batch x hidden_size]
+        :param vid_indices:
+        :return
+            decoder output
+        Note: we run this one step at a time i.e. you should use a outer loop
+            to process the whole sequence
+        '''
+
+        if self.do_flatten_parameters:
+            self.gru.flatten_parameters()
+
+        if self.discrete_representation:
+            word_embedded = self.embedding(motion_input).view(1, motion_input.size(0), -1)  # [1 x B x embedding_dim]
+            motion_input = self.dropout(word_embedded)
+        else:
+            motion_input = motion_input.view(1, motion_input.size(0), -1)  # [1 x batch x dim]
+
+        if debug:
+            print("Decoder_forward, motion_input", motion_input.shape) # [1, 128, 41])
+        if debug:
+            print("Decoder_forward, encoder_outputs", encoder_outputs.shape)  # [1, 128, 41])
+
+        # attention
+        if self.att_use:
+            attn_weights = self.attn(last_hidden[-1], encoder_outputs)  # [batch x 1 x T]
+            context = attn_weights.bmm(encoder_outputs.transpose(0, 1))  # [batch x 1 x attn_size]
+            context = context.transpose(0, 1)  # [1 x batch x attn_size]
+
+            # make input vec
+            rnn_input = torch.cat((motion_input, context), 2)  # [1 x batch x (dim + attn_size)]
+        else:
+            attn_weights = None
+            # rnn_input = torch.cat((motion_input, encoder_outputs), 2)  # [1 x batch x (dim + attn_size)]
+            rnn_input = motion_input
+        if debug:
+            print("Decoder_forward, rnn_input", rnn_input.shape) # [1, 128, 241])
+
+        # Check if unconditioned
+        if self.autoencoder_conditioned==False:
+            rnn_input = torch.zeros_like(rnn_input)
+        rnn_input = nn.Dropout(0.95)(rnn_input)
+
+        # if self.speaker_model:
+        #     assert vid_indices is not None
+        #     speaker_context = self.speaker_embedding(vid_indices).unsqueeze(0)
+        #     rnn_input = torch.cat((rnn_input, speaker_context), 2)  # [1 x batch x (dim + attn_size + embed_size)]
+
+        q = rnn_input.squeeze(0)
+        rnn_input = self.pre_linear(rnn_input.squeeze(0))
+        if debug:
+            print("Decoder_forward, rnn_input", rnn_input.shape) # torch.Size([128, 200])
+
+        rnn_input = rnn_input.unsqueeze(0)
+        if debug:
+            print("Decoder_forward, rnn_input", rnn_input.shape) # torch.Size([1, 128, 200])
+
+        # rnn
+
+        output, hidden = self.gru(rnn_input, last_hidden)
+
+        # post-fc
+        output = output.squeeze(0)  # [1 x batch x hidden_size] -> [batch x hidden_size]
+        output = self.out_layer(output)
+
+        return output, hidden, attn_weights
+
+
+class Generator(nn.Module):
+    def __init__(self, args, motion_dim, discrete_representation=False, speaker_model=None):
+        super(Generator, self).__init__()
+        self.output_size = motion_dim
+        self.n_layers = args.n_layers
+        self.discrete_representation = discrete_representation
+        self.decoder = BahdanauAttnDecoderRNN(args=args, input_size=args.rep_learning_dim,
+                                              hidden_size=args.hidden_size,
+                                              output_size=args.rep_learning_dim,
+                                              n_layers=self.n_layers,
+                                              dropout_p=args.dropout_prob,
+                                              discrete_representation=discrete_representation,
+                                              speaker_model=speaker_model)
+        self.is_training = True
+    def freeze_attn(self):
+        self.decoder.freeze_attn()
+
+    def forward(self, z, motion_input, last_hidden, encoder_output, vid_indices=None):
+        if z is None:
+            input_with_noise_vec = motion_input
+        else:
+            assert not self.discrete_representation  # not valid for discrete representation
+            input_with_noise_vec = torch.cat([motion_input, z], dim=1)  # [bs x (10+z_size)]
+
+        return self.decoder(input_with_noise_vec, last_hidden, encoder_output, vid_indices)
+
+
+class Autoencoder_VQVAE(nn.Module):
+    def __init__(self, args, pose_dim, n_frames):
+        super().__init__()
+
+        self.CNN = False
+
+
+        # 1. Defining Encoder and Decoder
+        self.encoder = EncoderRNN(
+            args.rep_learning_dim, args.hidden_size, args.n_layers,
+            dropout=args.dropout_prob, pre_trained_embedding=None)
+
+        self.out_layer_encoder = nn.Sequential(
+            nn.Linear(args.hidden_size, args.hidden_size),
+            nn.Tanh())
+        self.out_layer_decoder = nn.Sequential(
+            nn.Linear(args.hidden_size, pose_dim),
+            )
+
+        self.decoder = Generator(args, pose_dim, speaker_model=None)
+
+        if(self.CNN):
+            self.encoder = EncoderCNN(
+                args.rep_learning_dim, args.hidden_size, args.n_layers,
+                dropout=args.dropout_prob, pre_trained_embedding=None)
+            self.decoder = DecoderCNN(
+                args.hidden_size, args.hidden_size, args.n_layers,
+                dropout=args.dropout_prob, pre_trained_embedding=None)
+
+
+
+        # Defining Variational Autoencoder layers
+        if args.autoencoder_vae == 'True':
+            self.VAE = True
+            self.VAE_fc_mean = nn.Linear(self.decoder.n_layers*args.hidden_size,
+                                         self.decoder.n_layers*args.hidden_size)
+            self.VAE_fc_std = nn.Linear(self.decoder.n_layers*args.hidden_size,
+                                        self.decoder.n_layers*args.hidden_size)
+            self.VAE_fc_decoder = nn.Linear(self.decoder.n_layers*args.hidden_size,
+                                            self.decoder.n_layers*args.hidden_size)
+        else:
+            self.VAE = False
+        
+
+        # 2. Defining  Vector Quantizer layer
+        if args.autoencoder_vq == 'True':
+            decay = 0.85
+            self.vq = True
+            self.vq_components = int(args.autoencoder_vq_components)
+            self.commitment_cost = float(args.autoencoder_vq_commitment_cost)
+            # Todo: add a pre linear layer before feeding to the VQ layer
+
+            if decay > 0:
+                self.vq_layer = VQ_Payam_EMA(self.vq_components,
+                                                  args.hidden_size*args.n_layers, self.commitment_cost,
+                                                  decay)
+
+            else:
+                self.vq_layer = VQ_Payam(self.vq_components,
+                                                args.hidden_size*args.n_layers, self.commitment_cost)
+
+            self.vq_layer = VQ_Payam_GSSoft16(self.vq_components,
+                                         args.hidden_size * args.n_layers, self.commitment_cost)
+
+            # self.vq_layer  = VectorQuantGroup(n_channels=1,
+            #                                   n_classes=512,
+            #                                   vec_len=30,
+            #                                   num_group=16,
+            #                                   num_sample=32,
+            #                                   normalize=False)
+
+
+        else:
+            self.vq = False
+
+
+        # 3. Define parameters
+        self.n_frames = n_frames
+        self.n_pre_poses = args.n_pre_poses
+        self.pose_dim = args.rep_learning_dim
+
+        if args.autoencoder_conditioned == 'True':
+            self.autoencoder_conditioned = True
+        else:
+            self.autoencoder_conditioned = False
+
+
+
+
+        self.do = nn.Dropout(args.dropout_prob)
+
+        # Todo: should fix, currently we freez from train_epoch function
+        # if args.autoencoder_freeze_encoder=='True':
+        #     self.freez_encoder()
+
+    def freez_encoder(self):
+        self.autoencoder_fixed_encoder_weight = True
+
+        for param in self.encoder.parameters():
+            param.requires_grad = False
+        for param in self.VAE_fc_mean.parameters():
+            param.requires_grad = False
+        for param in self.VAE_fc_std.parameters():
+            param.requires_grad = False
+        for param in self.VAE_fc_decoder.parameters():
+            param.requires_grad = False
+
+    def freeze_VQminuse(self):
+        if self.VAE:
+            self.freez_encoder()
+        else:
+            for param in self.encoder.parameters():
+                param.requires_grad = False
+
+        for param in self.decoder.parameters():
+            param.requires_grad = False
+
+    def reparameterize(self, mu, logVar, train=True):
+
+        # Reparameterization takes in the input mu and logVar and sample the mu + std * eps
+        std = torch.exp(logVar / 2)
+        eps = torch.randn_like(std)
+        # Todo: fix this for train and test time checking
+        if train:
+            return mu + std * eps
+        else:
+            return mu # + std * eps
+
+    def forward(self, in_poses, out_poses, vq_layer_active=False):
+
+
+        # if self.vq and vq_layer_active:
+        #     self.freeze_VQminuse()
+
+
+        # reshape to (seq x batch x dim)
+        # in_text = in_text.transpose(0, 1)
+        in_poses = in_poses.transpose(0, 1)
+        in_poses = self.do(in_poses)
+        out_poses = out_poses.transpose(0, 1)
+
+        if not self.CNN:
+            outputs = torch.zeros(self.n_frames, out_poses.size(1), self.decoder.output_size).to(out_poses.device)
+
+        # run words through encoder
+        encoder_outputs, encoder_hidden = self.encoder(in_poses, None)
+
+        if self.CNN:
+            decoder_hidden = encoder_hidden
+        else:
+            decoder_hidden = (encoder_hidden[:self.decoder.n_layers])  # use last hidden state from encoder
+        # Todo: I put vector quantization here. However, we need to check it using both VAE and not VAE.
+        vq_layer_active = True
+        if self.vq and vq_layer_active:
+            loss_vq, quantized, perplexity_vq, encodings = self.vq_layer(decoder_hidden)
+            decoder_hidden = quantized
+        else:
+            loss_vq, quantized, perplexity_vq, encodings = torch.tensor(0), torch.tensor(0), \
+                                                           torch.tensor(0), torch.tensor(0)
+
+        # print("Sanity check:", encoder_hidden[:self.decoder.n_layers].shape)
+        # self.VAE = False
+        if self.VAE:
+            if debug:
+                print("self.VAE", self.VAE)
+            # decoder_hidden = encoder_hidden[:self.decoder.n_layers]  # use last hidden state from encoder
+            # [2, 128, 200]
+            # print("decoder_hidden!!! org", decoder_hidden.shape)
+            decoder_hidden = decoder_hidden.transpose(1, 0).contiguous() # [128, 2, 200]
+            decoder_hidden = torch.reshape(decoder_hidden, (decoder_hidden.shape[0], -1))
+            mean = self.VAE_fc_mean(decoder_hidden)
+            logvar = self.VAE_fc_std(decoder_hidden)
+            z = self.reparameterize(mean, logvar)
+            z = self.VAE_fc_decoder(z)
+            decoder_hidden = z.reshape(decoder_hidden.shape[0],
+                                       self.decoder.n_layers, -1)
+            decoder_hidden = decoder_hidden.transpose(1 ,0).contiguous()
+            # print("decoder_hidden!!! modified", decoder_hidden.shape)
+            decoder_first_hidden = decoder_hidden
+        # else:
+        #     decoder_hidden = encoder_hidden[:self.decoder.n_layers]  # use last hidden state from encoder
+        #     # print("decoder_hidden!!! not VAE ", decoder_hidden.shape)
+
+        if debug:
+            print("Decoder hidden from encoder:", decoder_hidden.shape)
+
+
+
+
+
+
+        decoder_first_hidden = decoder_hidden
+
+
+        # Todo: should be fixed such a parameter for inference time
+        all_hiddens = []
+
+
+        if self.CNN:
+            outputs, hidden = self.decoder(decoder_hidden)
+            shape = outputs.shape
+            # print(shape)
+            outputs = outputs.transpose(1,2)
+            # print(outputs.shape)
+            shape = outputs.shape
+            outputs = outputs.reshape(shape[0]*shape[1], shape[2])
+            # print(outputs.shape)
+            outputs = self.out_layer_decoder(outputs)
+            outputs = outputs.reshape(shape[0], shape[1], -1)
+            outputs = outputs.transpose(0, 1)
+            # print(outputs.shape)
+
+        else:
+            # run through decoder one time step at a time
+            decoder_input = out_poses[0]  # initial pose from the dataset
+            outputs[0] = decoder_input
+            for t in range(1, self.n_frames):
+                if not self.autoencoder_conditioned:
+                    decoder_input = torch.zeros_like(decoder_input)
+                decoder_output, decoder_hidden, _ = self.decoder(None, decoder_input, decoder_hidden, encoder_outputs,
+                                                                 None)
+                outputs[t] = decoder_output
+
+                if t < self.n_pre_poses:
+                    decoder_input = out_poses[t]  # next input is current target
+                else:
+                    decoder_input = decoder_output  # next input is current prediction
+                if not self.autoencoder_conditioned:
+                    decoder_input = torch.zeros_like(decoder_input)
+
+        if self.vq:
+            if self.VAE:
+                return outputs.transpose(0, 1), decoder_first_hidden[:self.decoder.n_layers], mean, logvar, loss_vq, perplexity_vq
+            else:
+                return outputs.transpose(0, 1), decoder_first_hidden[:self.decoder.n_layers], loss_vq, perplexity_vq
+        else:
+            if self.VAE:
+                return outputs.transpose(0, 1), decoder_first_hidden[:self.decoder.n_layers], mean, logvar
+            else:
+                return outputs.transpose(0, 1), decoder_first_hidden[:self.decoder.n_layers]
+
+
+class VQ_Payam(nn.Module):
+    def __init__(self, num_embeddings, embedding_dim, commitment_cost):
+        super(VQ_Payam, self).__init__()
+        self._embedding_dim = embedding_dim
+        self._num_embeddings = num_embeddings
+
+        self.pre_linear = nn.Linear(self._embedding_dim, self._embedding_dim)
+
+        self._embedding = nn.Embedding(self._num_embeddings, self._embedding_dim)
+        # self._embedding.weight.data.uniform_(-1 / self._num_embeddings, 1 / self._num_embeddings)
+        self._embedding.weight.data.uniform_(-0.2, 0.2 )
+        self._embedding.weight.data.normal_()
+
+        self._commitment_cost = commitment_cost
+
+        # self.embedding_grad(False)
+
+    def embedding_grad(self, what):
+        for param in self._embedding.parameters():
+            param.requires_grad = what
+
+    def forward(self, inputs):
+
+        loss_vq, loss, perplexity_vq, encodings = torch.tensor(0), torch.tensor(0), \
+                                                  torch.tensor(0), torch.tensor(0)
+        # 1. Pre-Linear
+        flat_input = inputs.view(-1, self._embedding_dim)
+        # flat_input = self.pre_linear(flat_input)
+
+
+        #__________________________
+        # 2. Calculate distances
+        distances = (torch.sum(flat_input ** 2, dim=1, keepdim=True)
+                     + torch.sum(self._embedding.weight ** 2, dim=1)
+                     - 2 * torch.matmul(flat_input, self._embedding.weight.t()))
+        if debug:
+            print("VQ_flat_input", flat_input.shape)
+            print("VQ_distances", distances.shape)
+
+        # 3. Find nearest encoding
+        encoding_indices = torch.argmin(distances, dim=1).unsqueeze(1)
+        # print("Idices: ", encoding_indices)
+
+        encodings = torch.zeros(encoding_indices.shape[0], self._num_embeddings, device=inputs.device)
+        encodings.scatter_(1, encoding_indices, 1)
+
+        # 4. Quantize and unflatten
+        quantized = torch.matmul(encodings, self._embedding.weight) #.view(input_shape)
+
+        m = torch.mean(quantized)
+        mw = torch.std(quantized)
+
+        quantized = torch.reshape(quantized, inputs.shape).contiguous()
+
+        torch.mean(quantized)
+
+        # 5. Loss
+        e_latent_loss = F.mse_loss(quantized.detach(), inputs)
+        q_latent_loss = F.mse_loss(quantized, inputs.detach())
+        # loss = q_latent_loss + self._commitment_cost * e_latent_loss
+        loss = q_latent_loss + self._commitment_cost * e_latent_loss
+
+        # loss = torch.square(loss)
+
+        quantized = inputs + (quantized - inputs).detach()
+        avg_probs = torch.mean(encodings, dim=0)
+        perplexity = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-10)))
+
+
+        return loss, quantized.contiguous(), perplexity, encodings
+        #___________________________
+
+
+
+        quantized = torch.reshape(flat_input, inputs.shape).contiguous()
+
+
+        return loss, quantized, perplexity_vq, encodings
+
+class VQ_Payam_EMA(nn.Module):
+    def __init__(self, num_embeddings, embedding_dim, commitment_cost, decay, epsilon=1e-5):
+        super(VQ_Payam_EMA, self).__init__()
+        self._embedding_dim = embedding_dim
+        self._num_embeddings = num_embeddings
+
+        self.pre_linear = nn.Linear(self._embedding_dim, self._embedding_dim)
+
+        self._embedding = nn.Embedding(self._num_embeddings, self._embedding_dim)
+        # self._embedding.weight.data.uniform_(-1 / self._num_embeddings, 1 / self._num_embeddings)
+        self._embedding.weight.data.uniform_(-1, 1)
+        # self._embedding.weight.data.normal_()
+
+        self._commitment_cost = commitment_cost
+
+
+        #     EMA
+        self.register_buffer('_ema_cluster_size', torch.zeros(num_embeddings))
+        self._ema_w = nn.Parameter(torch.Tensor(num_embeddings, self._embedding_dim))
+        self._ema_w.data.normal_()
+
+        self._decay = decay
+        self._epsilon = epsilon
+
+
+    def forward(self, inputs):
+
+
+        loss_vq, loss, perplexity_vq, encodings = torch.tensor(0), torch.tensor(0), \
+                                                  torch.tensor(0), torch.tensor(0)
+        # 1. Pre-Linear
+        flat_input = inputs.view(-1, self._embedding_dim)
+        flat_input = self.pre_linear(flat_input)
+
+
+        #__________________________
+        # 2. Calculate distances
+        distances = (torch.sum(flat_input ** 2, dim=1, keepdim=True)
+                     + torch.sum(self._embedding.weight ** 2, dim=1)
+                     - 2 * torch.matmul(flat_input, self._embedding.weight.t()))
+        if debug:
+            print("VQ_flat_input", flat_input.shape)
+            print("VQ_distances", distances.shape)
+
+        # 3. Find nearest encoding
+        encoding_indices = torch.argmin(distances, dim=1).unsqueeze(1)
+        # print("Idices: ", encoding_indices)
+        encodings = torch.zeros(encoding_indices.shape[0], self._num_embeddings, device=inputs.device)
+        encodings.scatter_(1, encoding_indices, 1)
+
+        # Todo:
+        # Test modification for soft vq
+        # ...
+
+        # 4. Quantize and unflatten
+        quantized = torch.matmul(encodings, self._embedding.weight) #.view(input_shape)
+        quantized = torch.reshape(quantized, inputs.shape).contiguous()
+
+        # 5.1 Use EMA to update the embedding vectors
+        if self.training:
+            self._ema_cluster_size = self._ema_cluster_size * self._decay + \
+                                     (1 - self._decay) * torch.sum(encodings, 0)
+
+            # Laplace smoothing of the cluster size
+            n = torch.sum(self._ema_cluster_size.data)
+            self._ema_cluster_size = (
+                    (self._ema_cluster_size + self._epsilon)
+                    / (n + self._num_embeddings * self._epsilon) * n)
+
+            dw = torch.matmul(encodings.t(), flat_input)
+            self._ema_w = nn.Parameter(self._ema_w * self._decay + (1 - self._decay) * dw)
+
+            self._embedding.weight = nn.Parameter(self._ema_w / self._ema_cluster_size.unsqueeze(1))
+
+
+        # 5. Loss
+        e_latent_loss = F.mse_loss(quantized.detach(), inputs)
+        # q_latent_loss = F.mse_loss(quantized, inputs.detach())
+        # loss = q_latent_loss + self._commitment_cost * e_latent_loss
+        # loss = q_latent_loss + self._commitment_cost * e_latent_loss
+        loss = self._commitment_cost * e_latent_loss
+        # loss = torch.square(loss)
+
+        quantized = inputs + (quantized - inputs).detach()
+        avg_probs = torch.mean(encodings, dim=0)
+        perplexity = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-10)))
+
+
+        return loss, quantized.contiguous(), perplexity, encodings
+        #___________________________
+
+
+
+        quantized = torch.reshape(flat_input, inputs.shape).contiguous()
+
+
+        return loss, quantized, perplexity_vq, encodings
+
+
+
+class VQ_Payam_GSSoft(nn.Module):
+    def __init__(self, num_embeddings, embedding_dim, commitment_cost):
+        super(VQ_Payam_GSSoft, self).__init__()
+        self._embedding_dim = embedding_dim
+        self._num_embeddings = num_embeddings
+
+        self.pre_linear = nn.Linear(self._embedding_dim, self._embedding_dim)
+
+        self._embedding = nn.Embedding(self._num_embeddings, self._embedding_dim)
+        self._embedding.weight.data.uniform_(-1 / self._num_embeddings, 1 / self._num_embeddings)
+        # self._embedding.weight.data.uniform_(-0.2, 0.2)
+        self._embedding.weight.data.normal_()
+
+        self._commitment_cost = commitment_cost
+
+        # self.embedding_grad(False)
+
+        self.mean_layer = nn.Linear(embedding_dim, embedding_dim)
+        self.logvar_layer = nn.Linear(embedding_dim, num_embeddings)
+
+
+    def embedding_grad(self, what):
+        for param in self._embedding.parameters():
+            param.requires_grad = what
+
+    def reparameterize(self, mu, logVar, train=True):
+
+        # Reparameterization takes in the input mu and logVar and sample the mu + std * eps
+        std = torch.exp(logVar / 2)
+        eps = torch.randn_like(std)
+        # Todo: fix this for train and test time checking
+        if train:
+            return mu + std * eps
+        else:
+            return mu # + std * eps
+    def soft_prob(self, dist, smooth):
+        dist = (dist)/400
+        # print("Dist", dist)
+        # print("Smooth", smooth)
+        # zarb = torch.multiply(dist, 0.5 * smooth)
+        # print("torch.multiply(dist, 0.5 * smooth)", zarb)
+        # print(torch.exp(-zarb))
+        # print(dist.shape, smooth.shape)
+
+        # print((-torch.multiply(dist, 0.5 * smooth)))
+        # print(dist)
+        prob = torch.exp(-torch.multiply(dist, 0.5 * smooth)) / torch.sqrt(smooth)
+
+        # print(prob)
+        # sumprobsumprob = prob.sum(1)
+        # print("sum_pron", sumprobsumprob.shape, prob.shape)
+        # print(prob.sum(1).unsqueeze(1).expand(prob.shape[0], prob.shape[1]).shape)
+        # print("Mean", prob.sum(0))
+        probs = prob / prob.sum(1).unsqueeze(1).expand(prob.shape[0], prob.shape[1])
+        # print(probs.shape)
+        # print("Normalized Prob", probs)
+
+        return probs
+
+    def forward(self, inputs):
+
+        loss_vq, loss, perplexity_vq, encodings = torch.tensor(0), torch.tensor(0), \
+                                                  torch.tensor(0), torch.tensor(0)
+        # 1. Pre-Linear
+        flat_input = inputs.view(-1, self._embedding_dim)
+        # flat_input = self.pre_linear(flat_input)
+
+        flat_input = self.mean_layer(flat_input)
+        z_logvar = self.logvar_layer(flat_input)
+
+        # __________________________
+        # 2. Calculate distances
+        distances = (torch.sum(flat_input ** 2, dim=1, keepdim=True)
+                     + torch.sum(self._embedding.weight ** 2, dim=1)
+                     - 2 * torch.matmul(flat_input, self._embedding.weight.t()))
+        if debug:
+            print("VQ_flat_input", flat_input.shape)
+            print("VQ_distances", distances.shape)
+
+        # # 3. Find nearest encoding
+        # encoding_indices = torch.argmin(distances, dim=1).unsqueeze(1)
+        # # print("Idices: ", encoding_indices)
+        # encodings = torch.zeros(encoding_indices.shape[0], self._num_embeddings, device=inputs.device)
+        # encodings.scatter_(1, encoding_indices, 1)
+
+        smooth = 1. / torch.exp(z_logvar) ** 2
+        probs = self.soft_prob(distances, smooth)
+        encodings = probs
+        # print("---1--", probs.shape, probs)
+
+        # 4. Quantize and unflatten
+        quantized = torch.matmul(encodings, self._embedding.weight)  # .view(input_shape)
+
+
+        quantized = torch.reshape(quantized, inputs.shape).contiguous()
+
+
+
+
+        # 5. Loss
+        e_latent_loss = F.mse_loss(quantized.detach(), inputs)
+        q_latent_loss = F.mse_loss(quantized, inputs.detach())
+        # loss = q_latent_loss + self._commitment_cost * e_latent_loss
+        loss = q_latent_loss + self._commitment_cost * e_latent_loss
+
+        # loss = torch.square(loss)
+
+        quantized = inputs + (quantized - inputs).detach()
+        avg_probs = torch.mean(encodings, dim=0)
+        perplexity = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-10)))
+
+        return loss, quantized.contiguous(), perplexity, encodings
+        # ___________________________
+
+        quantized = torch.reshape(flat_input, inputs.shape).contiguous()
+
+        return loss, quantized, perplexity_vq, encodings
+
+
+class VQ_Payam_GSSoft16(nn.Module):
+    def __init__(self, num_embeddings, embedding_dim, commitment_cost):
+        super(VQ_Payam_GSSoft16, self).__init__()
+        self._embedding_dim = embedding_dim
+        self._num_embeddings = num_embeddings
+
+        self.reduced_dim = 16
+        self.pre_linear = nn.Linear(self._embedding_dim, self.reduced_dim)
+        self.post_linear = nn.Linear(self.reduced_dim, self._embedding_dim)
+
+        self._embedding = nn.Embedding(self._num_embeddings, self.reduced_dim)
+        self._embedding.weight.data.uniform_(-1 / self._num_embeddings, 1 / self._num_embeddings)
+        # self._embedding.weight.data.uniform_(-0.2, 0.2)
+        self._embedding.weight.data.normal_()
+
+        self._commitment_cost = commitment_cost
+
+        # self.embedding_grad(False)
+
+        self.mean_layer = nn.Linear(self.reduced_dim, self.reduced_dim)
+        self.logvar_layer = nn.Linear(self.reduced_dim, num_embeddings)
+
+
+    def embedding_grad(self, what):
+        for param in self._embedding.parameters():
+            param.requires_grad = what
+
+    def reparameterize(self, mu, logVar, train=True):
+
+        # Reparameterization takes in the input mu and logVar and sample the mu + std * eps
+        std = torch.exp(logVar / 2)
+        eps = torch.randn_like(std)
+        # Todo: fix this for train and test time checking
+        if train:
+            return mu + std * eps
+        else:
+            return mu # + std * eps
+    def soft_prob(self, dist, smooth):
+        dist = (dist)/400
+        # print("Dist", dist)
+        # print("Smooth", smooth)
+        # zarb = torch.multiply(dist, 0.5 * smooth)
+        # print("torch.multiply(dist, 0.5 * smooth)", zarb)
+        # print(torch.exp(-zarb))
+        # print(dist.shape, smooth.shape)
+
+        # print((-torch.multiply(dist, 0.5 * smooth)))
+        # print(dist)
+        prob = torch.exp(-torch.multiply(dist, 0.5 * smooth)) / torch.sqrt(smooth)
+
+        # print(prob)
+        # sumprobsumprob = prob.sum(1)
+        # print("sum_pron", sumprobsumprob.shape, prob.shape)
+        # print(prob.sum(1).unsqueeze(1).expand(prob.shape[0], prob.shape[1]).shape)
+        # print("Mean", prob.sum(0))
+        probs = prob / prob.sum(1).unsqueeze(1).expand(prob.shape[0], prob.shape[1])
+        # print(probs.shape)
+        # print("Normalized Prob", probs)
+
+
+
+
+        return probs
+
+    def forward(self, inputs):
+
+        loss_vq, loss, perplexity_vq, encodings = torch.tensor(0), torch.tensor(0), \
+                                                  torch.tensor(0), torch.tensor(0)
+        # 1. Pre-Linear
+        flat_input = inputs.view(-1, self._embedding_dim)
+        flat_input = self.pre_linear(flat_input)
+
+        flat_input = self.mean_layer(flat_input)
+        z_logvar = self.logvar_layer(flat_input)
+
+        # __________________________
+        # 2. Calculate distances
+        distances = (torch.sum(flat_input ** 2, dim=1, keepdim=True)
+                     + torch.sum(self._embedding.weight ** 2, dim=1)
+                     - 2 * torch.matmul(flat_input, self._embedding.weight.t()))
+        if debug:
+            print("VQ_flat_input", flat_input.shape)
+            print("VQ_distances", distances.shape)
+
+        # # 3. Find nearest encoding
+        # encoding_indices = torch.argmin(distances, dim=1).unsqueeze(1)
+        # # print("Idices: ", encoding_indices)
+        # encodings = torch.zeros(encoding_indices.shape[0], self._num_embeddings, device=inputs.device)
+        # encodings.scatter_(1, encoding_indices, 1)
+
+        smooth = 1. / torch.exp(z_logvar) ** 2
+        probs = self.soft_prob(distances, smooth)
+        encodings = probs
+        # print("---1--", probs.shape, probs)
+
+        # 4. Quantize and unflatten
+        quantized = torch.matmul(encodings, self._embedding.weight)  # .view(input_shape)
+
+        quantized = self.post_linear(quantized)
+        quantized = torch.reshape(quantized, inputs.shape).contiguous()
+
+
+
+
+        # 5. Loss
+        e_latent_loss = F.mse_loss(quantized.detach(), inputs)
+        q_latent_loss = F.mse_loss(quantized, inputs.detach())
+        # loss = q_latent_loss + self._commitment_cost * e_latent_loss
+        loss = q_latent_loss + self._commitment_cost * e_latent_loss
+
+        # loss = torch.square(loss)
+
+        quantized = inputs + (quantized - inputs).detach()
+
+        encodings_HARD = encodings.argmax(dim=1)
+        avg_probs = torch.mean(encodings, dim=0)
+
+
+        perplexity = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-10)))
+
+        return loss, quantized.contiguous(), perplexity, encodings
+        # ___________________________
+
+        quantized = torch.reshape(flat_input, inputs.shape).contiguous()
+
+        return loss, quantized, perplexity_vq, encodings
+
+
+
+class VectorQuantizer(nn.Module):
+    def __init__(self, num_embeddings, embedding_dim, commitment_cost):
+        super(VectorQuantizer, self).__init__()
+
+        self._embedding_dim = embedding_dim
+        self._num_embeddings = num_embeddings
+
+        self.pre_lin = nn.Linear(self._embedding_dim, self._embedding_dim)
+
+        self._embedding = nn.Embedding(self._num_embeddings, self._embedding_dim)
+        if debug:
+            print("VQ_Embedding", self._embedding)
+        self._embedding.weight.data.uniform_(-1 / self._num_embeddings, 1 / self._num_embeddings)
+        self._commitment_cost = commitment_cost
+
+    def forward(self, inputs):
+
+        loss_vq, loss, perplexity_vq, encodings = torch.tensor(0), torch.tensor(0), \
+                                                  torch.tensor(0), torch.tensor(0)
+        return loss, inputs, perplexity_vq, encodings
+
+
+        org = inputs.detach().clone()
+        # convert inputs from BCHW -> BHWC
+        # Convert input from Layers, Batch, HDim -> Batch, Layers.HDim
+        # Convert input from 2     , 128  , 200 -> 128   , 400
+        # inputs = inputs.permute(0, 2, 3, 1).contiguous()
+        # Todo: fix this to n layers (currently for two layers)
+        first_shape = inputs.shape
+        inputs = torch.hstack((inputs[0], inputs[1]))
+        input_shape = inputs.shape
+        if debug:
+            print("VQ input_shape:", input_shape)
+
+
+
+        # inputs = self.pre_lin(inputs)
+        # Flatten input
+        flat_input = inputs.view(-1, self._embedding_dim)
+        if debug:
+            print("VQ_flat_input", flat_input.shape)
+
+
+        loss_vq, loss, perplexity_vq, encodings = torch.tensor(0), torch.tensor(0), \
+                                                  torch.tensor(0), torch.tensor(0)
+        quantized = torch.reshape(inputs, first_shape).contiguous()
+
+        xxx = torch.sum(org -quantized)
+
+        return loss, quantized, perplexity_vq, encodings
+
+        # Calculate distances
+        if debug:
+            print("(torch.sum(flat_input ** 2, dim=1, keepdim=True)", torch.sum(flat_input ** 2, dim=1, keepdim=True).shape)
+            print("torch.sum(self._embedding.weight ** 2, dim=1)", torch.sum(self._embedding.weight ** 2, dim=1).shape)
+            print("2 * torch.matmul(flat_input, self._embedding.weight.t()))", (2 * torch.matmul(flat_input, self._embedding.weight.t())).shape)
+        distances = (torch.sum(flat_input ** 2, dim=1, keepdim=True)
+                     + torch.sum(self._embedding.weight ** 2, dim=1)
+                     - 2 * torch.matmul(flat_input, self._embedding.weight.t()))
+        if debug:
+            print("VQ_flat_input", flat_input.shape)
+            print("VQ_distances", distances.shape)
+
+        # Encoding
+        encoding_indices = torch.argmin(distances, dim=1).unsqueeze(1)
+
+        # print("Idices: ", encoding_indices)
+
+        encodings = torch.zeros(encoding_indices.shape[0], self._num_embeddings, device=inputs.device)
+        encodings.scatter_(1, encoding_indices, 1)
+
+        # Quantize and unflatten
+        quantized = torch.matmul(encodings, self._embedding.weight).view(input_shape)
+
+        # Loss
+        e_latent_loss = F.mse_loss(quantized.detach(), inputs)
+        q_latent_loss = F.mse_loss(quantized, inputs.detach())
+        loss = q_latent_loss + self._commitment_cost * e_latent_loss
+
+
+        quantized = inputs + (quantized - inputs).detach()
+        avg_probs = torch.mean(encodings, dim=0)
+        perplexity = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-10)))
+
+
+        # Todo: to sanity check
+        # loss = 0
+        # quantized = inputs
+
+
+        # convert quantized from BHWC -> BCHW
+
+        # from Layers, Batch, HDim -> Batch, Layers.HDim
+        # quantized.permute(0, 3, 1, 2)
+        # Convert input from 2     , 128  , 200 -> 128   , 400
+        if debug:
+            print("VQ_quantized:", quantized.shape)
+        # Todo: is it ok to reshape like this.
+        quantized = torch.reshape(quantized, (2, quantized.shape[0], -1)).contiguous()
+        if debug:
+            print("VQ_quantized:", quantized.shape)
+
+
+
+        return loss, quantized, perplexity, encodings
+
+
+class VectorQuantizerEMA(nn.Module):
+    def __init__(self, num_embeddings, embedding_dim, commitment_cost, decay, epsilon=1e-5):
+        super(VectorQuantizerEMA, self).__init__()
+
+        self._embedding_dim = embedding_dim
+        self._num_embeddings = num_embeddings
+
+        self.pre_lin = nn.Linear(self._embedding_dim, self._embedding_dim)
+
+        self._embedding = nn.Embedding(self._num_embeddings, self._embedding_dim)
+        self._embedding.weight.data.normal_()
+        self._commitment_cost = commitment_cost
+
+        self.register_buffer('_ema_cluster_size', torch.zeros(num_embeddings))
+        self._ema_w = nn.Parameter(torch.Tensor(num_embeddings, self._embedding_dim))
+        self._ema_w.data.normal_()
+
+        self._decay = decay
+        self._epsilon = epsilon
+
+    def forward(self, inputs):
+        # convert inputs from BCHW -> BHWC
+        # inputs = inputs.permute(0, 2, 3, 1).contiguous()
+        inputs = torch.hstack((inputs[0], inputs[1]))
+        input_shape = inputs.shape
+
+        inputs = self.pre_lin(inputs)
+
+        # Flatten input
+        flat_input = inputs.view(-1, self._embedding_dim)
+
+        # Calculate distances
+        distances = (torch.sum(flat_input ** 2, dim=1, keepdim=True)
+                     + torch.sum(self._embedding.weight ** 2, dim=1)
+                     - 2 * torch.matmul(flat_input, self._embedding.weight.t()))
+
+        # Encoding
+        encoding_indices = torch.argmin(distances, dim=1).unsqueeze(1)
+        encodings = torch.zeros(encoding_indices.shape[0], self._num_embeddings, device=inputs.device)
+        encodings.scatter_(1, encoding_indices, 1)
+
+        # Quantize and unflatten
+        quantized = torch.matmul(encodings, self._embedding.weight).view(input_shape)
+
+        # Use EMA to update the embedding vectors
+        if self.training:
+            self._ema_cluster_size = self._ema_cluster_size * self._decay + \
+                                     (1 - self._decay) * torch.sum(encodings, 0)
+
+            # Laplace smoothing of the cluster size
+            n = torch.sum(self._ema_cluster_size.data)
+            self._ema_cluster_size = (
+                    (self._ema_cluster_size + self._epsilon)
+                    / (n + self._num_embeddings * self._epsilon) * n)
+
+            dw = torch.matmul(encodings.t(), flat_input)
+            self._ema_w = nn.Parameter(self._ema_w * self._decay + (1 - self._decay) * dw)
+
+            self._embedding.weight = nn.Parameter(self._ema_w / self._ema_cluster_size.unsqueeze(1))
+
+        # Loss
+        e_latent_loss = F.mse_loss(quantized.detach(), inputs)
+        loss = self._commitment_cost * e_latent_loss
+
+        # Straight Through Estimator
+        quantized = inputs + (quantized - inputs).detach()
+        avg_probs = torch.mean(encodings, dim=0)
+        perplexity = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-10)))
+
+        # convert quantized from BHWC -> BCHW
+
+        quantized = torch.reshape(quantized, (2, quantized.shape[0], -1)).contiguous()
+
+        return loss, quantized, perplexity, encodings
+
+
+class VectorQuantGroup(nn.Module):
+    """
+        Input: (N, samples, n_channels, vec_len) numeric tensor
+        Output: (N, samples, n_channels, vec_len) numeric tensor
+    """
+
+    def __init__(self, n_channels, n_classes, vec_len, num_group, num_sample, normalize=False):
+        super().__init__()
+        self._commitment_cost = 0.25
+        if normalize:
+            target_scale = 0.06
+            self.embedding_scale = target_scale
+            self.normalize_scale = target_scale
+        else:
+            self.embedding_scale = 1e-3
+            self.normalize_scale = None
+
+        self.n_classes = n_classes
+        self._num_group = num_group
+        self._num_sample = num_sample
+        if not self.n_classes % self._num_group == 0:
+            raise ValueError('num of embeddings in each group should be an integer')
+        self._num_classes_per_group = int(self.n_classes / self._num_group)
+
+        # self.embedding0 = nn.Parameter(
+        #     torch.randn(n_channels, n_classes, vec_len, requires_grad=True) * self.embedding_scale)
+        self._num_embeddings = n_classes
+        self._embedding_dim = 400
+        self._embedding = nn.Embedding(self._num_embeddings, self._embedding_dim)
+
+        self.offset = torch.arange(n_channels).cuda() * n_classes
+        # self.offset: (n_channels) long tensor
+        self.after_update()
+
+    def forward(self, x0):
+        if self.normalize_scale:
+            target_norm = self.normalize_scale * math.sqrt(x0.size(3))
+            x = target_norm * x0 / x0.norm(dim=3, keepdim=True)
+            embedding = target_norm * self._embedding / self._embedding.norm(dim=2, keepdim=True)
+        else:
+            x = x0
+            embedding = self._embedding
+        # logger.log(f'std[x] = {x.std()}')
+        x1 = x.reshape(-1, self._embedding_dim)
+        # x1: (N*samples, n_channels, 1, vec_len) numeric tensor
+
+        # Perform chunking to avoid overflowing GPU RAM.
+        index_chunks = []
+        prob_chunks = []
+        for x1_chunk in x1.split(512, dim=0):
+            # d = (x1_chunk - embedding).norm(dim=3)
+            d = (torch.sum(x1 ** 2, dim=1, keepdim=True)
+             + torch.sum(embedding.weight ** 2, dim=1)
+             - 2 * torch.matmul(x1, embedding.weight.t()))
+
+            # Compute the group-wise distance
+            d_group = torch.zeros(x1_chunk.shape[0], self._num_group).to(torch.device('cuda'))
+            for i in range(self._num_group):
+                d_group[:, i] = torch.mean(
+                    d[:, i * self._num_classes_per_group: (i + 1) * self._num_classes_per_group], 1)
+            degrup_numpy = d_group.detach().cpu().numpy()
+            # Find the nearest group
+            index_chunk_group = d_group.argmin(dim=1).unsqueeze(1)
+
+            # Generate mask for the nearest group
+            index_chunk_group = index_chunk_group.repeat(1, self._num_classes_per_group)
+            index_chunk_group = torch.mul(self._num_classes_per_group, index_chunk_group)
+            idx_mtx = torch.LongTensor([x for x in range(self._num_classes_per_group)]).unsqueeze(0).cuda()
+            index_chunk_group += idx_mtx
+            showindex = index_chunk_group.detach().cpu().numpy()
+            encoding_mask = torch.zeros(x1_chunk.shape[0], self.n_classes).cuda()
+            encoding_mask.scatter_(1, index_chunk_group, 1)
+            showencoding = encoding_mask.detach().cpu().numpy()
+
+            # Todo: fixed up to here!
+
+            # Compute the weight atoms in the group
+            # Comment: Why should we use the squeeze()
+            encoding_prob = torch.div(1, d.squeeze())
+
+            # Apply the mask
+            masked_encoding_prob = torch.mul(encoding_mask, encoding_prob)
+            p, idx = masked_encoding_prob.sort(dim=1, descending=True)
+            prob_chunks.append(p[:, :self._num_sample])
+            index_chunks.append(idx[:, :self._num_sample])
+
+        index = torch.cat(index_chunks, dim=0)
+        prob_dist = torch.cat(prob_chunks, dim=0)
+        prob_dist = F.normalize(prob_dist, p=1, dim=1)
+        # index: (N*samples, n_channels) long tensor
+        if True:  # compute the entropy
+            hist = index[:, 0].float().cpu().histc(bins=self.n_classes, min=-0.5, max=self.n_classes - 0.5)
+            prob = hist.masked_select(hist > 0) / len(index)
+            entropy = - (prob * prob.log()).sum().item()
+            # logger.log(f'entrypy: {entropy:#.4}/{math.log(self.n_classes):#.4}')
+        else:
+            entropy = 0
+        entropy = torch.tensor(entropy)
+        index1 = (index + self.offset)
+        # index1: (N*samples*n_channels) long tensor
+        output_list = []
+        for i in range(self._num_sample):
+            output_list.append(torch.mul(embedding.weight.view(-1, embedding.weight.size(1)).index_select(dim=0, index=index1[:, i]),
+                                         prob_dist[:, i].unsqueeze(1).detach()))
+
+        output_cat = torch.stack(output_list, dim=2)
+        output_flat = torch.sum(output_cat, dim=2)
+        # output_flat: (N*samples*n_channels, vec_len) numeric tensor
+        output = output_flat.view(x.size())
+
+        out0 = (output - x).detach() + x
+        out1 = F.mse_loss( x.detach() , output) #.float().norm(dim=2).pow(2)
+        out2 = F.mse_loss(x , output.detach()) #.float()) #.norm(dim=2).pow(2) + (x - x0).float().norm(dim=2).pow(2)
+        # logger.log(f'std[embedding0] = {self.embedding0.view(-1, embedding.size(2)).index_select(dim=0, index=index1).std()}')
+        # return (out0, out1, out2, entropy)
+
+        quantized = out0
+        e_latent_loss = out2.mean()
+        q_latent_loss = out1.mean()
+        # Loss
+        # e_latent_loss = F.mse_loss(quantized.detach(), inputs)
+        # q_latent_loss = F.mse_loss(quantized, inputs.detach())
+        loss = q_latent_loss + self._commitment_cost * e_latent_loss
+
+        # quantized = inputs + (quantized - inputs).detach()
+        # avg_probs = torch.mean(encodings, dim=0)
+        # perplexity = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-10)))
+        perplexity = entropy
+        encodings = encoding_prob.unsqueeze(0)
+        return loss, quantized, perplexity, encodings
+
+        discrete, vq_pen, encoder_pen, entropy = self.vq(continuous.unsqueeze(2))
+        # discrete: (N, 14, 1, 64)
+    def after_update(self):
+        if self.normalize_scale:
+            with torch.no_grad():
+                target_norm = self.embedding_scale * math.sqrt(self._embedding.size(2))
+                self._embedding.mul_(target_norm / self._embedding.norm(dim=2, keepdim=True))
\ No newline at end of file
diff --git a/scripts/model/Autoencoder_model.py b/scripts/model/Autoencoder_model.py
new file mode 100644
index 0000000..3e3e5a4
--- /dev/null
+++ b/scripts/model/Autoencoder_model.py
@@ -0,0 +1,388 @@
+import torch
+from torch.autograd import Variable
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+
+'''
+Based on the following Se2Seq implementations:
+- https://github.com/AuCson/PyTorch-Batch-Attention-Seq2seq
+- https://github.com/spro/practical-pytorch/blob/master/seq2seq-translation/seq2seq-translation-batched.ipynb
+'''
+
+debug = False
+
+class EncoderRNN(nn.Module):
+    def __init__(self, input_size, hidden_size, n_layers=1, dropout=0.5, pre_trained_embedding=None):
+        super(EncoderRNN, self).__init__()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.n_layers = n_layers
+        self.dropout = dropout
+
+        self.in_layer = nn.Linear(input_size, hidden_size)
+        self.gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=self.dropout, bidirectional=True)
+
+        self.do_flatten_parameters = False
+        if torch.cuda.device_count() > 1:
+            self.do_flatten_parameters = True
+
+    def forward(self, input_seqs, hidden=None):
+        '''
+        :param input_seqs:
+            Variable of shape (num_step(T),batch_size(B)), sorted decreasingly by lengths(for packing)
+        :param input_lengths:
+            list of sequence length
+        :param hidden:
+            initial state of GRU
+        :returns:
+            GRU outputs in shape (T,B,hidden_size(H))
+            last hidden stat of RNN(i.e. last output for GRU)
+        '''
+        if self.do_flatten_parameters:
+            self.gru.flatten_parameters()
+        if debug:
+            print("Encoder_Forward_input", input_seqs.shape)
+        input_seq_in_layered = self.in_layer(input_seqs)
+        outputs, hidden = self.gru(input_seq_in_layered, hidden)
+        # outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(outputs)  # unpack (back to padded)
+        outputs = outputs[:, :, :self.hidden_size] + outputs[:, :, self.hidden_size:]  # Sum bidirectional outputs
+        if debug:
+            print("Encoder_Forward_ out, hidden", outputs.shape, hidden.shape)
+        return outputs, hidden
+
+
+class Attn(nn.Module):
+    def __init__(self, hidden_size):
+        super(Attn, self).__init__()
+        self.hidden_size = hidden_size
+        self.attn = nn.Linear(self.hidden_size * 2, hidden_size)
+        self.v = nn.Parameter(torch.rand(hidden_size))
+        stdv = 1. / math.sqrt(self.v.size(0))
+        self.v.data.normal_(mean=0, std=stdv)
+
+    def forward(self, hidden, encoder_outputs):
+        '''
+        :param hidden:
+            previous hidden state of the decoder, in shape (layers*directions,B,H)
+        :param encoder_outputs:
+            encoder outputs from Encoder, in shape (T,B,H)
+        :return
+            attention energies in shape (B,T)
+        '''
+        max_len = encoder_outputs.size(0)
+        this_batch_size = encoder_outputs.size(1)
+        H = hidden.repeat(max_len, 1, 1).transpose(0, 1)
+        encoder_outputs = encoder_outputs.transpose(0, 1)  # [B*T*H]
+        attn_energies = self.score(H, encoder_outputs)  # compute attention score
+        return F.softmax(attn_energies, dim=1).unsqueeze(1)  # normalize with softmax
+
+    def score(self, hidden, encoder_outputs):
+        energy = torch.tanh(self.attn(torch.cat([hidden, encoder_outputs], 2)))  # [B*T*2H]->[B*T*H]
+        energy = energy.transpose(2, 1)  # [B*H*T]
+        v = self.v.repeat(encoder_outputs.data.shape[0], 1).unsqueeze(1)  # [B*1*H]
+        energy = torch.bmm(v, energy)  # [B*1*T]
+        return energy.squeeze(1)  # [B*T]
+
+
+class BahdanauAttnDecoderRNN(nn.Module):
+    def __init__(self, args, input_size, hidden_size, output_size, n_layers=1, dropout_p=0.1,
+                 discrete_representation=False, speaker_model=None):
+        super(BahdanauAttnDecoderRNN, self).__init__()
+
+        # define parameters
+        self.hidden_size = hidden_size
+        self.output_size = output_size
+        self.n_layers = n_layers
+        self.dropout_p = dropout_p
+        self.discrete_representation = discrete_representation
+        self.speaker_model = speaker_model
+
+        # define embedding layer
+        if self.discrete_representation:
+            self.embedding = nn.Embedding(output_size, hidden_size)
+            self.dropout = nn.Dropout(dropout_p)
+
+        # if self.speaker_model:
+        #     self.speaker_embedding = nn.Embedding(speaker_model.n_words, 8)
+
+        # calc input size
+        # if self.discrete_representation:
+        #     input_size = hidden_size  # embedding size
+        # linear_input_size = input_size + hidden_size
+        # if self.speaker_model:
+        #     linear_input_size += 8
+
+        if args.autoencoder_conditioned == 'True':
+            self.autoencoder_conditioned = True
+        else:
+            self.autoencoder_conditioned = False
+
+        # define layers
+
+        if args.autoencoder_att == 'True':
+            self.attn = Attn(hidden_size)
+            self.att_use = True
+        else:
+            self.att_use = False
+
+        if self.att_use:
+            linear_input_size = input_size + hidden_size
+        else:
+            linear_input_size = input_size
+        self.pre_linear = nn.Sequential(
+            nn.Linear(linear_input_size, hidden_size),
+            nn.BatchNorm1d(hidden_size),
+            nn.ReLU(inplace=True)
+        )
+
+
+        self.gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=dropout_p)
+        if args.autoencoder_fixed_weight == 'True':
+            self.autoencoder_fixed_weight = True
+            for param in self.gru.parameters():
+                param.requires_grad = False
+
+
+
+        # self.out = nn.Linear(hidden_size * 2, output_size)
+        self.out_layer = nn.Linear(hidden_size, output_size)
+
+        self.do_flatten_parameters = False
+        if torch.cuda.device_count() > 1:
+            self.do_flatten_parameters = True
+
+    def freeze_attn(self):
+        for param in self.attn.parameters():
+            param.requires_grad = False
+
+    def forward(self, motion_input, last_hidden, encoder_outputs, vid_indices=None):
+        '''
+        :param motion_input:
+            motion input for current time step, in shape [batch x dim]
+        :param last_hidden:
+            last hidden state of the decoder, in shape [layers x batch x hidden_size]
+        :param encoder_outputs:
+            encoder outputs in shape [steps x batch x hidden_size]
+        :param vid_indices:
+        :return
+            decoder output
+        Note: we run this one step at a time i.e. you should use a outer loop
+            to process the whole sequence
+        '''
+
+        if self.do_flatten_parameters:
+            self.gru.flatten_parameters()
+
+        if self.discrete_representation:
+            word_embedded = self.embedding(motion_input).view(1, motion_input.size(0), -1)  # [1 x B x embedding_dim]
+            motion_input = self.dropout(word_embedded)
+        else:
+            motion_input = motion_input.view(1, motion_input.size(0), -1)  # [1 x batch x dim]
+
+        if debug:
+            print("Decoder_forward, motion_input", motion_input.shape) # [1, 128, 41])
+        if debug:
+            print("Decoder_forward, encoder_outputs", encoder_outputs.shape)  # [1, 128, 41])
+
+        # attention
+        if self.att_use:
+            attn_weights = self.attn(last_hidden[-1], encoder_outputs)  # [batch x 1 x T]
+            context = attn_weights.bmm(encoder_outputs.transpose(0, 1))  # [batch x 1 x attn_size]
+            context = context.transpose(0, 1)  # [1 x batch x attn_size]
+
+            # make input vec
+            rnn_input = torch.cat((motion_input, context), 2)  # [1 x batch x (dim + attn_size)]
+        else:
+            attn_weights = None
+            # rnn_input = torch.cat((motion_input, encoder_outputs), 2)  # [1 x batch x (dim + attn_size)]
+            rnn_input = motion_input
+        if debug:
+            print("Decoder_forward, rnn_input", rnn_input.shape) # [1, 128, 241])
+
+        # Check if unconditioned
+        # Todo: Oh my god
+        # print(self.autoencoder_conditioned)
+        if self.autoencoder_conditioned==False:
+            rnn_input = torch.zeros_like(rnn_input)
+
+        # if self.speaker_model:
+        #     assert vid_indices is not None
+        #     speaker_context = self.speaker_embedding(vid_indices).unsqueeze(0)
+        #     rnn_input = torch.cat((rnn_input, speaker_context), 2)  # [1 x batch x (dim + attn_size + embed_size)]
+
+        q = rnn_input.squeeze(0)
+        rnn_input = self.pre_linear(rnn_input.squeeze(0))
+        if debug:
+            print("Decoder_forward, rnn_input", rnn_input.shape) # torch.Size([128, 200])
+
+        rnn_input = rnn_input.unsqueeze(0)
+        if debug:
+            print("Decoder_forward, rnn_input", rnn_input.shape) # torch.Size([1, 128, 200])
+
+        # rnn
+
+        output, hidden = self.gru(rnn_input, last_hidden)
+
+        # post-fc
+        output = output.squeeze(0)  # [1 x batch x hidden_size] -> [batch x hidden_size]
+        output = self.out_layer(output)
+
+        return output, hidden, attn_weights
+
+
+class Generator(nn.Module):
+    def __init__(self, args, motion_dim, discrete_representation=False, speaker_model=None):
+        super(Generator, self).__init__()
+        self.output_size = motion_dim
+        self.n_layers = args.n_layers
+        self.discrete_representation = discrete_representation
+        self.decoder = BahdanauAttnDecoderRNN(args=args, input_size=args.rep_learning_dim,
+                                              hidden_size=args.hidden_size,
+                                              output_size=args.rep_learning_dim,
+                                              n_layers=self.n_layers,
+                                              dropout_p=args.dropout_prob,
+                                              discrete_representation=discrete_representation,
+                                              speaker_model=speaker_model)
+        self.is_training = True
+    def freeze_attn(self):
+        self.decoder.freeze_attn()
+
+    def forward(self, z, motion_input, last_hidden, encoder_output, vid_indices=None):
+        if z is None:
+            input_with_noise_vec = motion_input
+        else:
+            assert not self.discrete_representation  # not valid for discrete representation
+            input_with_noise_vec = torch.cat([motion_input, z], dim=1)  # [bs x (10+z_size)]
+
+        return self.decoder(input_with_noise_vec, last_hidden, encoder_output, vid_indices)
+
+
+class Autoencoder_seq2seq(nn.Module):
+    def __init__(self, args, pose_dim, n_frames):
+        super().__init__()
+        self.encoder = EncoderRNN(
+            args.rep_learning_dim, args.hidden_size, args.n_layers,
+            dropout=args.dropout_prob, pre_trained_embedding=None)
+        self.decoder = Generator(args, pose_dim, speaker_model=None)
+
+        self.n_frames = n_frames
+        self.n_pre_poses = args.n_pre_poses
+        self.pose_dim = args.rep_learning_dim
+
+        if args.autoencoder_conditioned == 'True':
+            self.autoencoder_conditioned = True
+        else:
+            self.autoencoder_conditioned = False
+
+        try:
+            if args.autoencoder_vae == 'True':
+                self.VAE = True
+                self.VAE_fc_mean = nn.Linear(self.decoder.n_layers*args.hidden_size,
+                                             self.decoder.n_layers*args.hidden_size)
+                self.VAE_fc_std = nn.Linear(self.decoder.n_layers*args.hidden_size,
+                                            self.decoder.n_layers*args.hidden_size)
+                self.VAE_fc_decoder = nn.Linear(self.decoder.n_layers*args.hidden_size,
+                                                self.decoder.n_layers*args.hidden_size)
+            else:
+                self.VAE = False
+        except:
+            self.VAE = False
+
+        self.do = nn.Dropout(args.dropout_prob)
+
+        # Todo: should fix, currently we freez from train_epoch function
+        # if args.autoencoder_freeze_encoder=='True':
+        #     self.freez_encoder()
+
+    def freez_encoder(self):
+        self.autoencoder_fixed_encoder_weight = True
+
+        for param in self.encoder.parameters():
+            param.requires_grad = False
+        for param in self.VAE_fc_mean.parameters():
+            param.requires_grad = False
+        for param in self.VAE_fc_std.parameters():
+            param.requires_grad = False
+        for param in self.VAE_fc_decoder.parameters():
+            param.requires_grad = False
+
+
+
+    def reparameterize(self, mu, logVar, train=True):
+
+        # Reparameterization takes in the input mu and logVar and sample the mu + std * eps
+        std = torch.exp(logVar / 2)
+        eps = torch.randn_like(std)
+        # Todo: fix this for train and test time checking
+        if train:
+            return mu + std * eps
+        else:
+            return mu # + std * eps
+
+    def forward(self, in_poses, out_poses):
+        # reshape to (seq x batch x dim)
+        # in_text = in_text.transpose(0, 1)
+        in_poses = in_poses.transpose(0, 1)
+        in_poses = self.do(in_poses)
+        out_poses = out_poses.transpose(0, 1)
+
+        outputs = torch.zeros(self.n_frames, out_poses.size(1), self.decoder.output_size).to(out_poses.device)
+
+        # run words through encoder
+        encoder_outputs, encoder_hidden = self.encoder(in_poses, None)
+
+
+
+        # print("Sanity check:", encoder_hidden[:self.decoder.n_layers].shape)
+        # self.VAE = False
+        if self.VAE:
+            if debug:
+                print("self.VAE", self.VAE)
+            decoder_hidden = encoder_hidden[:self.decoder.n_layers]  # use last hidden state from encoder
+            # [2, 128, 200]
+            # print("decoder_hidden!!! org", decoder_hidden.shape)
+            decoder_hidden = decoder_hidden.transpose(1, 0).contiguous() # [128, 2, 200]
+            decoder_hidden = torch.reshape(decoder_hidden, (decoder_hidden.shape[0], -1))
+            mean = self.VAE_fc_mean(decoder_hidden)
+            logvar = self.VAE_fc_std(decoder_hidden)
+            z = self.reparameterize(mean, logvar)
+            z = self.VAE_fc_decoder(z)
+            decoder_hidden = z.reshape(decoder_hidden.shape[0],
+                                       self.decoder.n_layers, -1)
+            decoder_hidden = decoder_hidden.transpose(1 ,0).contiguous()
+            # print("decoder_hidden!!! modified", decoder_hidden.shape)
+            decoder_first_hidden = decoder_hidden
+        else:
+            decoder_hidden = encoder_hidden[:self.decoder.n_layers]  # use last hidden state from encoder
+            # print("decoder_hidden!!! not VAE ", decoder_hidden.shape)
+
+        decoder_first_hidden = decoder_hidden
+        # run through decoder one time step at a time
+        decoder_input = out_poses[0]  # initial pose from the dataset
+        outputs[0] = decoder_input
+
+        # Todo: should be fixed such a parameter for inference time
+        all_hiddens = []
+
+
+
+        for t in range(1, self.n_frames):
+            decoder_output, decoder_hidden, _ = self.decoder(None, decoder_input, decoder_hidden, encoder_outputs,
+                                                             None)
+            outputs[t] = decoder_output
+
+            if t < self.n_pre_poses:
+                decoder_input = out_poses[t]  # next input is current target
+            else:
+                decoder_input = decoder_output  # next input is current prediction
+            if not self.autoencoder_conditioned:
+                decoder_input = torch.zeros_like(decoder_input)
+
+        if self.VAE:
+            return outputs.transpose(0, 1),\
+                   decoder_first_hidden[:self.decoder.n_layers],\
+                   mean, logvar
+        else:
+            return outputs.transpose(0, 1),\
+                   decoder_first_hidden[:self.decoder.n_layers]
diff --git a/scripts/model/DAE_model.py b/scripts/model/DAE_model.py
new file mode 100644
index 0000000..4f05555
--- /dev/null
+++ b/scripts/model/DAE_model.py
@@ -0,0 +1,470 @@
+import torch
+from torch.autograd import Variable
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+from torch.distributions import Categorical, RelaxedOneHotCategorical
+
+class DAE_Network(nn.Module):
+    def __init__(self, motion_dim, latent_dim):
+        super(DAE_Network, self).__init__()
+        print("init DAE");
+
+        self.dropout = nn.Dropout(0.2)
+
+        if latent_dim == -1: # To skip this network for ablation-study
+            self.encoder = None
+            self.decoder = None
+            return
+        if latent_dim == -2: # Linear transformation
+            self.encoder = nn.Sequential(nn.Linear(motion_dim, 200),)
+            self.decoder = nn.Sequential(nn.Linear(200, motion_dim),)
+            self.dropout = nn.Dropout(0.3)
+            return
+
+        self.encoder = nn.Sequential(
+            nn.Linear(motion_dim, latent_dim),
+            nn.ReLU(),
+            # nn.Linear(2 * latent_dim, latent_dim),
+            # nn.Tanh(),
+            # nn.Linear(32, 32),
+            # nn.Tanh(),
+        )
+
+        self.decoder = nn.Sequential(
+            # nn.Linear(latent_dim, 2* latent_dim),
+            # nn.Tanh(),
+            nn.Linear( latent_dim, motion_dim),
+            # nn.Tanh(),
+            # nn.Linear(100, motion_dim),
+
+        )
+
+
+    def forward(self, x, get_latent=False):
+
+        CHECK = x.cpu().detach().numpy()
+
+        if self.encoder == None:
+            if get_latent:
+                return x, x
+            return x
+
+        # print("_________________")
+        # print(self.encoder)
+        # print(x.shape)
+        # print("_________________")
+        x = torch.squeeze(x)
+        x_np = x.cpu().detach().numpy()
+        # print(x.shape)A regular autoencoder (similar to part1) with a bottleneck size of 30 dimensions, and Tanh activation function, and turning some of the inputs values to zero.
+        x = self.dropout(x)
+        x = self.encoder(x)
+        latent = x.detach().clone()
+        # print("Encoded", x.shape)
+        x = self.decoder(x)
+        x = torch.unsqueeze(x, 2)
+        # print("Decoder", x.shape)
+        if get_latent:
+            return x, latent
+        else:
+            return x
+
+# Vector quantization at the frame level
+class VQ_Frame(nn.Module):
+    def __init__(self, motion_dim, latent_dim, vae, vq_components):
+        super(VQ_Frame, self).__init__()
+        print("init VQ_DAE");
+
+
+        self.encoder = nn.Sequential(
+            nn.Linear(motion_dim, latent_dim),
+            # nn.Tanh(),
+            # nn.Linear(motion_dim-10, motion_dim-30),
+            # nn.Tanh(),
+        )
+        nn.init.xavier_normal_(self.encoder[0].weight)
+
+        # Todo: test
+        self.bachnorm = nn.BatchNorm1d(latent_dim)
+
+        self.skip_vq = False
+        self.vq_components = vq_components
+        self.commitment_cost = 0.25
+        decay = 0.99
+        if decay > 0.0:
+            self.vq_layer = VQ_Payam_EMA(self.vq_components,
+                                     latent_dim, self.commitment_cost, decay)
+        else:
+            self.vq_layer = VQ_Payam(self.vq_components,
+                                     latent_dim, self.commitment_cost)
+
+
+        # GS-Soft Vector Quantization
+        self.gs_soft = False
+        if self.gs_soft:
+            self.vq_layer = VQ_Payam_GSOFT(self.vq_components,
+                                         latent_dim, self.commitment_cost, decay)
+            self.log_scale = nn.Parameter(torch.Tensor([0.0]))
+
+
+        self.decoder = nn.Sequential(
+            # nn.Linear(motion_dim-30, motion_dim-10),
+            # nn.Tanh(),
+            nn.Linear(latent_dim, motion_dim),
+        )
+        nn.init.xavier_normal_(self.decoder[0].weight)
+        self.dropout = nn.Dropout()
+
+        self.vae = vae
+        if self.vae:
+            self.VAE_fc_mean = nn.Linear(latent_dim, latent_dim)
+            self.VAE_fc_std = nn.Linear(latent_dim, latent_dim)
+            self.VAE_fc_decoder = nn.Linear(latent_dim, latent_dim)
+
+
+
+    def reparameterize(self, mu, logVar, train=True):
+
+        # Reparameterization takes in the input mu and logVar and sample the mu + std * eps
+        std = torch.exp(logVar / 2)
+        eps = torch.randn_like(std)
+        # Todo: fix this for train and test time checking
+        if train:
+            return mu + std * eps
+        else:
+            return mu # + std * eps
+
+    def forward(self, x, Inference=False):
+        # print("_________________")
+        # print(self.encoder)
+        # print(x.shape)
+        # print("_________________")
+        # 1.
+        x = torch.squeeze(x)
+        # print(x.shape)
+        x = self.dropout(x)
+        x = self.encoder(x)
+        x = self.bachnorm(x)
+        # 2.
+        latent = x.detach().clone()
+        # print("Encoded", x.shape)
+
+        # 4.1 VAE
+        if self.vae == True:
+            mean = self.VAE_fc_mean(x)
+            logvar = self.VAE_fc_std(x)
+            z = self.reparameterize(mean, logvar)
+            z = self.VAE_fc_decoder(z)
+            x = z
+
+
+        # 3. Vector Quantization
+        if not self.skip_vq:
+            loss_vq, quantized, perplexity_vq, encodings = self.vq_layer(x)
+            x = quantized
+        else:
+            loss_vq, perplexity_vq = torch.tensor(0), torch.tensor(0)
+        # 4. Decoding
+
+
+        x = self.decoder(x)
+        x = torch.unsqueeze(x, 2)
+
+
+
+        # print("Decoder", x.shape)
+        if Inference:
+            return x, latent, encodings
+        else:
+            if self.gs_soft:
+                scale = torch.exp(self.log_scale)
+                x = torch.distributions.Normal(x, scale)
+            if self.vae:
+                return x, loss_vq, perplexity_vq, logvar, mean
+            else:
+                return x, loss_vq, perplexity_vq
+
+
+class VQ_Payam(nn.Module):
+    def __init__(self, num_embeddings, embedding_dim, commitment_cost):
+        super(VQ_Payam, self).__init__()
+        self._embedding_dim = embedding_dim
+        self._num_embeddings = num_embeddings
+
+        self.pre_linear = nn.Linear(self._embedding_dim, self._embedding_dim)
+
+        self._embedding = nn.Embedding(self._num_embeddings, self._embedding_dim)
+        self._embedding.weight.data.uniform_(-1 / self._num_embeddings, 1 / self._num_embeddings)
+
+        self._commitment_cost = commitment_cost
+
+    def forward(self, inputs):
+
+        loss_vq, loss, perplexity_vq, encodings = torch.tensor(0), torch.tensor(0), \
+                                                  torch.tensor(0), torch.tensor(0)
+        # 1. Pre-Linear
+        flat_input = inputs.view(-1, self._embedding_dim)
+        # flat_input = self.pre_linear(flat_input)
+
+        # __________________________
+        # 2. Calculate distances
+        distances = (torch.sum(flat_input ** 2, dim=1, keepdim=True)
+                     + torch.sum(self._embedding.weight ** 2, dim=1)
+                     - 2 * torch.matmul(flat_input, self._embedding.weight.t()))
+        debug = False
+        if debug:
+            print("VQ_flat_input", flat_input.shape)
+            print("VQ_distances", distances.shape)
+
+        # 3. Find nearest encoding
+        encoding_indices = torch.argmin(distances, dim=1).unsqueeze(1)
+
+        # for ind in encoding_indices:
+        #     print("Idices: ", ind)
+
+        encodings = torch.zeros(encoding_indices.shape[0], self._num_embeddings, device=inputs.device)
+        encodings.scatter_(1, encoding_indices, 1)
+
+        # 4. Quantize and unflatten
+        quantized = torch.matmul(encodings, self._embedding.weight)  # .view(input_shape)
+        quantized = torch.reshape(quantized, inputs.shape).contiguous()
+
+        # 5. Loss
+        e_latent_loss = F.mse_loss(quantized.detach(), inputs)
+        q_latent_loss = F.mse_loss(quantized, inputs.detach())
+        loss =  q_latent_loss +  self._commitment_cost * e_latent_loss
+
+        quantized = inputs + (quantized - inputs).detach()
+        avg_probs = torch.mean(encodings, dim=0)
+        perplexity = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-10)))
+        return loss, quantized.contiguous(), perplexity, encodings
+        # ___________________________
+
+
+        quantized = torch.reshape(flat_input, inputs.shape).contiguous()
+
+        return loss, quantized, perplexity_vq, encodings
+
+class VQ_Payam_EMA(nn.Module):
+    def __init__(self, num_embeddings, embedding_dim, commitment_cost,
+                 decay, epsilon=1e-5):
+        super(VQ_Payam_EMA, self).__init__()
+        self._embedding_dim = embedding_dim
+        self._num_embeddings = num_embeddings
+
+        self.pre_linear = nn.Linear(self._embedding_dim, self._embedding_dim)
+
+        self._embedding = nn.Embedding(self._num_embeddings, self._embedding_dim)
+        self._embedding.weight.data.uniform_(-1 / self._num_embeddings, 1 / self._num_embeddings)
+
+        self._commitment_cost = commitment_cost
+
+        #EMA
+        self.register_buffer('_ema_cluster_size', torch.zeros(num_embeddings))
+        self._ema_w = nn.Parameter(torch.Tensor(num_embeddings, self._embedding_dim))
+        self._ema_w.data.normal_()
+
+        self._decay = decay
+        self._epsilon = epsilon
+
+    def forward(self, inputs):
+
+        loss_vq, loss, perplexity_vq, encodings = torch.tensor(0), torch.tensor(0), \
+                                                  torch.tensor(0), torch.tensor(0)
+        # 1. Pre-Linear
+        flat_input = inputs.view(-1, self._embedding_dim)
+        # flat_input = self.pre_linear(flat_input)
+
+        # __________________________
+        # 2. Calculate distances
+        distances = (torch.sum(flat_input ** 2, dim=1, keepdim=True)
+                     + torch.sum(self._embedding.weight ** 2, dim=1)
+                     - 2 * torch.matmul(flat_input, self._embedding.weight.t()))
+        debug = False
+        if debug:
+            print("VQ_flat_input", flat_input.shape)
+            print("VQ_distances", distances.shape)
+
+        # 3. Find nearest encoding
+        encoding_indices = torch.argmin(distances, dim=1).unsqueeze(1)
+
+        # for ind in encoding_indices:
+        #     print("Idices: ", ind)
+
+        encodings = torch.zeros(encoding_indices.shape[0], self._num_embeddings, device=inputs.device)
+        encodings.scatter_(1, encoding_indices, 1)
+
+        # 4. Quantize and unflatten
+        quantized = torch.matmul(encodings, self._embedding.weight)  # .view(input_shape)
+        quantized = torch.reshape(quantized, inputs.shape).contiguous()
+
+        # X. Use EMA to update the embedding vectors
+        if self.training:
+            self._ema_cluster_size = self._ema_cluster_size * self._decay + \
+                                     (1 - self._decay) * torch.sum(encodings, 0)
+
+            # Laplace smoothing of the cluster size
+            n = torch.sum(self._ema_cluster_size.data)
+            self._ema_cluster_size = (
+                    (self._ema_cluster_size + self._epsilon)
+                    / (n + self._num_embeddings * self._epsilon) * n)
+
+            dw = torch.matmul(encodings.t(), flat_input)
+            self._ema_w = nn.Parameter(self._ema_w * self._decay + (1 - self._decay) * dw)
+
+            self._embedding.weight = nn.Parameter(self._ema_w / self._ema_cluster_size.unsqueeze(1))
+
+
+
+        # 5. Loss
+        e_latent_loss = F.mse_loss(quantized.detach(), inputs)
+        # q_latent_loss = F.mse_loss(quantized, inputs.detach())
+        loss = self._commitment_cost * e_latent_loss
+
+        # 6. Straight Through Estimator
+        quantized = inputs + (quantized - inputs).detach()
+        avg_probs = torch.mean(encodings, dim=0)
+        perplexity = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-10)))
+        return loss, quantized.contiguous(), perplexity, encodings
+        # ___________________________
+
+
+        quantized = torch.reshape(flat_input, inputs.shape).contiguous()
+
+        return loss, quantized, perplexity_vq, encodings
+
+class VQ_Payam_GSOFT(nn.Module):
+    def __init__(self, num_embeddings, embedding_dim, commitment_cost,
+                 decay, epsilon=1e-5):
+        super(VQ_Payam_GSOFT, self).__init__()
+        self._embedding_dim = embedding_dim
+        self._num_embeddings = num_embeddings
+
+        self.pre_linear = nn.Linear(self._embedding_dim, self._embedding_dim)
+
+        self._embedding = nn.Embedding(self._num_embeddings, self._embedding_dim)
+        self._embedding.weight.data.uniform_(-1 / self._num_embeddings, 1 / self._num_embeddings)
+
+        self._commitment_cost = commitment_cost
+
+
+
+
+
+    def forward(self, inputs):
+
+        loss_vq, loss, perplexity_vq, encodings = torch.tensor(0), torch.tensor(0), \
+                                                  torch.tensor(0), torch.tensor(0)
+        # 1. Pre-Linear
+        flat_input = inputs.view(-1, self._embedding_dim)
+        # flat_input = self.pre_linear(flat_input)
+
+        # __________________________
+        # 2. Calculate distances
+        distances = (torch.sum(flat_input ** 2, dim=1, keepdim=True)
+                     + torch.sum(self._embedding.weight ** 2, dim=1)
+                     - 2 * torch.matmul(flat_input, self._embedding.weight.t()))
+
+        dist = RelaxedOneHotCategorical(0.5, logits=-distances)
+        encodings = dist.rsample()  # .view(N, -1, M)
+        if self.training:
+            encodings = dist.rsample() # .view(N=latend_dim, -1, M=#embedding)
+        else:
+            # 3. Find nearest encoding
+            encoding_indices = torch.argmax(dist.probs, dim=1).unsqueeze(1)
+            print(encoding_indices)
+            encodings = torch.zeros(encoding_indices.shape[0], self._num_embeddings, device=inputs.device)
+            encodings.scatter_(1, encoding_indices, 1)
+
+            # encodings = dist.rsample()
+
+        # 4. Quantize and unflatten
+        quantized = torch.matmul(encodings, self._embedding.weight)  # .view(input_shape)
+        # print(encodings.shape, encodings.sum(), encodings)
+        quantized = torch.reshape(quantized, inputs.shape).contiguous()
+
+
+
+
+        # 5. Loss
+        KL = dist.probs * (dist.logits + math.log(self._num_embeddings))
+        # print("KL_Shape", KL.shape) #128 * 100
+        KL[(dist.probs == 0).expand_as(KL)] = 0
+        KL = KL.sum(dim=0)
+        # print("KL.sum(dim=0)", KL.shape)
+        KL = KL.mean()
+        # print("Kl.mean()", KL.shape, KL)
+
+
+        # 6. Straight Through Estimator
+        quantized = inputs + (quantized - inputs).detach()
+        avg_probs = torch.mean(encodings, dim=0)
+        perplexity = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-10)))
+
+        return KL, quantized.contiguous(), perplexity, encodings
+        # ___________________________
+        quantized = torch.reshape(flat_input, inputs.shape).contiguous()
+        return loss, quantized, perplexity_vq, encodings
+
+class VAE_Network(nn.Module):
+    def __init__(self, motion_dim, latent_dim):
+        super(VAE_Network, self).__init__()
+        print("init VAE_Network");
+        self.encoder = nn.Sequential(
+            nn.Linear(motion_dim, latent_dim),
+            nn.Tanh(),
+            # nn.Linear(motion_dim-10, motion_dim-30),
+            # nn.Tanh(),
+        )
+
+        self.decoder = nn.Sequential(
+            # nn.Linear(motion_dim-30, motion_dim-10),
+            # nn.Tanh(),
+            nn.Linear(latent_dim, motion_dim),
+
+        )
+        self.dropout = nn.Dropout()
+
+        self.vae=True
+        self.VAE_fc_mean = nn.Linear(latent_dim, latent_dim)
+        self.VAE_fc_std = nn.Linear(latent_dim, latent_dim)
+        self.VAE_fc_decoder = nn.Linear(latent_dim, latent_dim)
+
+    def reparameterize(self, mu, logVar, train=True):
+
+        # Reparameterization takes in the input mu and logVar and sample the mu + std * eps
+        std = torch.exp(logVar / 2)
+        eps = torch.randn_like(std)
+        # Todo: fix this for train and test time checking
+        if train:
+            return mu + std * eps
+        else:
+            return mu  # + std * eps
+
+    def forward(self, x, get_latent=False):
+        # print("_________________")
+        # print(self.encoder)
+        # print(x.shape)
+        # print("_________________")
+        x = torch.squeeze(x)
+        # print(x.shape)A regular autoencoder (similar to part1) with a bottleneck size of 30 dimensions, and Tanh activation function, and turning some of the inputs values to zero.
+        x = self.dropout(x)
+        x = self.encoder(x)
+        latent = x.detach().clone()
+        # print("Encoded", x.shape)
+
+        # VAE
+        mean = self.VAE_fc_mean(x)
+        logvar = self.VAE_fc_std(x)
+        z = self.reparameterize(mean, logvar)
+        z = self.VAE_fc_decoder(z)
+        x = z
+
+
+        x = self.decoder(x)
+        x = torch.unsqueeze(x, 2)
+        # print("Decoder", x.shape)
+        if get_latent:
+            return x, latent
+        else:
+            return x, logvar, mean
\ No newline at end of file
diff --git a/scripts/model/Helper_models.py b/scripts/model/Helper_models.py
new file mode 100644
index 0000000..9ffc896
--- /dev/null
+++ b/scripts/model/Helper_models.py
@@ -0,0 +1,667 @@
+import torch
+import torch.nn as nn
+import numpy as np
+from torch.nn import functional as F
+
+
+
+class WavEncoder(nn.Module):
+    def __init__(self):
+        super().__init__()
+
+        # self.feat_extractor = nn.Sequential(
+        self.al1 = nn.Conv1d(1, 16, 15, stride=5, padding=1600)
+        self.al2 = nn.BatchNorm1d(16)
+        self.al3 = nn.LeakyReLU(0.3, inplace=True)
+        self.al4 = nn.Conv1d(16, 32, 15, stride=6)
+        self.al5 = nn.BatchNorm1d(32)
+        self.al6 = nn.LeakyReLU(0.3, inplace=True)
+        self.al7 = nn.Conv1d(32, 64, 15, stride=6)
+        self.al8 = nn.BatchNorm1d(64)
+        self.al9 = nn.LeakyReLU(0.3, inplace=True)
+        self.al10 = nn.Conv1d(64, 128, 20, stride=6)
+        self.al11 = nn.Conv1d(128, 200, 15, stride=8)
+        # )
+
+    def forward(self, wav_data):
+        wav_data = wav_data.unsqueeze(1)  # add channel dim
+        # out = self.feat_extractor(wav_data)
+        out1 = self.al1(wav_data)
+        out2 = self.al2(out1)
+        out3 = self.al3(out2)
+        out4 = self.al4(out3)
+        out5 = self.al5(out4)
+        out6 = self.al6(out5)
+        out7 = self.al7(out6)
+        out8 = self.al8(out7)
+        out9 = self.al9(out8)
+        out10 = self.al10(out9)
+        out11 = self.al11(out10)
+
+        return out11.transpose(1, 2)  # to (batch x seq x dim)
+
+def spec_conv1d(n_layer=3, n_channel=[64, 32, 16, 8], filter_size=[1, 3, 3], stride=[1, 2, 2]):
+    """
+    Construction of conv. layers. Note the current implementation always effectively turn to 1-D conv,
+    inspired by https://arxiv.org/pdf/1704.04222.pdf.
+    :param n_layer: number of conv. layers
+    :param n_channel: in/output number of channels for each layer ( len(n_channel) = n_layer + 1 ).
+            The first channel is the number of freqeuncy bands of input spectrograms
+    :param filter_size: the filter size (x-axis) for each layer ( len(filter_size) = n_layer )
+    :param stride: filter stride size (x-axis) for each layer ( len(stride) = n_layer )
+    :return: an object (nn.Sequential) constructed of specified conv. layers
+    TODO:
+        [x] directly use nn.Conv1d for implementation
+        [] allow different activations and batch normalization functions
+    """
+
+    assert len(n_channel) == n_layer + 1, "This must fulfill: len(n_channel) = n_layer + 1"
+    ast_msg = "The following must fulfill: len(filter_size) == len(stride) == n_layer"
+    assert len(filter_size) == len(stride) == n_layer, ast_msg
+
+    # construct layers
+    conv_layers = []
+    for i in range(n_layer):
+        in_channel, out_channel = n_channel[i:i + 2]
+        conv_layers += [
+            nn.Conv1d(in_channel, out_channel, filter_size[i], stride[i]),
+            nn.ReLU(),
+            nn.BatchNorm1d(out_channel)
+        ]
+
+    return nn.Sequential(*conv_layers)
+
+def fc(n_layer, n_channel, activation='tanh', batchNorm=True):
+    """
+    Construction of fc. layers.
+    :param n_layer: number of fc. layers
+    :param n_channel: in/output number of neurons for each layer ( len(n_channel) = n_layer + 1 )
+    :param activation: allow either 'tanh' or None for now
+    :param batchNorm: True|False, indicate apply batch normalization or not
+    TODO:
+        [] allow different activations and batch normalization functions
+    """
+
+    assert len(n_channel) == n_layer + 1, "This must fulfill: len(n_channel) = n_layer + 1"
+    assert activation in [None, 'tanh'], "Only implement 'tanh' for now"
+
+    fc_layers = []
+    for i in range(n_layer):
+        layer = [nn.Linear(n_channel[i], n_channel[i + 1])]
+        if batchNorm:
+            layer.append(nn.BatchNorm1d(n_channel[i + 1]))
+        if activation:
+            layer.append(nn.Tanh())
+        fc_layers += layer
+
+    return nn.Sequential(*fc_layers)
+
+class WavEncoder2(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.input_size = [128, 32]
+        self.n_freqBand, self.n_contextWin = self.input_size
+        n_convChannel = [32, 16, 8]
+        n_convLayer = 3
+        filter_size = [1, 3, 3]
+        stride = [1, 2, 2]
+        n_fcLayer = 1
+        n_fcChannel = [200]
+        self.encoder = spec_conv1d(n_convLayer, [self.n_freqBand] + n_convChannel, filter_size, stride)
+        self.flat_size, self.encoder_outputSize = self._infer_flat_size()
+        self.encoder_fc = fc(n_fcLayer, [self.flat_size, *n_fcChannel], activation='tanh', batchNorm=True)
+
+    def _infer_flat_size(self):
+        encoder_output = self.encoder(torch.ones(1, *self.input_size))
+        return int(np.prod(encoder_output.size()[1:])), encoder_output.size()[1:]
+
+    def forward(self, x):
+        h = self.encoder(x)
+        h2 = self.encoder_fc(h.view(-1, self.flat_size))
+
+        return h2 #.transpose(1, 2)  # to (batch x seq x dim)
+
+
+Audio_Features = True
+both = False
+class EncoderRNN_With_Audio(nn.Module):
+    def __init__(self, input_size, embed_size, hidden_size, n_layers=1, dropout=0.5, pre_trained_embedding=None):
+        super(EncoderRNN_With_Audio, self).__init__()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.embed_size = embed_size
+        self.n_layers = n_layers
+        self.dropout = dropout
+
+        if pre_trained_embedding is not None:  # use pre-trained embedding (e.g., word2vec, glove)
+            assert pre_trained_embedding.shape[0] == input_size
+            assert pre_trained_embedding.shape[1] == embed_size
+            self.embedding = nn.Embedding.from_pretrained(torch.FloatTensor(pre_trained_embedding), freeze=False)
+        else:
+            self.embedding = nn.Embedding(input_size, embed_size)
+
+        self.gru = nn.GRU(embed_size, hidden_size, n_layers, dropout=self.dropout, bidirectional=True)
+
+
+        if Audio_Features:
+            self.combine_lin = nn.Linear(embed_size * 2, embed_size)
+            self.audio_encoder = WavEncoder2()
+            self.gru = nn.GRU(hidden_size, hidden_size, n_layers,
+                              dropout=self.dropout, bidirectional=True)
+        if both:
+            self.audio_encoder = WavEncoder_tri()
+            self.gru = nn.GRU(embed_size+hidden_size, hidden_size, n_layers,
+                              dropout=self.dropout, bidirectional=True)
+
+
+        self.do_flatten_parameters = False
+        if torch.cuda.device_count() > 1:
+            self.do_flatten_parameters = True
+
+    def forward(self, input_text, input_lengths, input_audio, hidden=None):
+        '''
+        :param input_text:
+            Variable of shape (num_step(T),batch_size(B)), sorted decreasingly by lengths(for packing)
+        :param input_lengths:
+            list of sequence length
+        :param hidden:
+            initial state of GRU
+        :returns:
+            GRU outputs in shape (T,B,hidden_size(H))
+            last hidden stat of RNN(i.e. last output for GRU)
+        '''
+        if self.do_flatten_parameters:
+            self.gru.flatten_parameters()
+        '''
+
+        embedded = self.embedding(input_text)
+        packed = torch.nn.utils.rnn.pack_padded_sequence(embedded, input_lengths)
+        '''
+
+        if Audio_Features or both:
+            # Todo: we may need do somepreprocessing.
+            audio_shape = input_audio.shape # [B, Seq_lenght, 1D-signal]
+
+            audio_raw_for_now = False
+            if  audio_raw_for_now:
+                input_audio = torch.reshape(input_audio, (-1, audio_shape[2])) # [B*Seq_len, 1D_signal]
+
+            else:
+                input_audio = torch.reshape(input_audio, (-1, audio_shape[-2],  # [B*Seq_len, freq_band, chunk_len]
+                                                          audio_shape[-1]))
+            audio_feat = self.audio_encoder(input_audio)
+            audio_feat = torch.reshape(audio_feat, (audio_shape[1], audio_shape[0], -1 ))
+            if both:
+                embedded = self.embedding(input_text)
+                packed = torch.nn.utils.rnn.pack_padded_sequence(embedded, input_lengths)
+                packed = torch.cat((packed, audio_feat), dim=2)
+            # packed = self.combine_lin(packed)
+            else:
+                packed = audio_feat
+
+        outputs, hidden = self.gru(packed, hidden)
+        # outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(outputs)  # unpack (back to padded)
+        outputs = outputs[:, :, :self.hidden_size] + outputs[:, :, self.hidden_size:]  # Sum bidirectional outputs
+        return outputs, hidden
+
+
+
+
+
+# ----------------------------------------------------------
+# Trimodal paper
+# ----------------------------------------------------------
+
+class WavEncoder_tri(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.feat_extractor = nn.Sequential(
+            nn.Conv1d(1, 16, 15, stride=5, padding=1600),
+            nn.BatchNorm1d(16),
+            nn.LeakyReLU(0.3, inplace=True),
+            nn.Conv1d(16, 32, 15, stride=6),
+            nn.BatchNorm1d(32),
+            nn.LeakyReLU(0.3, inplace=True),
+            nn.Conv1d(32, 64, 15, stride=6),
+            nn.BatchNorm1d(64),
+            nn.LeakyReLU(0.3, inplace=True),
+            nn.Conv1d(64, 32, 15, stride=6),
+        )
+        self.out_layer = nn.Linear(32*15, 200)
+    def forward(self, wav_data):
+        wav_data = wav_data.unsqueeze(1)  # add channel dim
+        out = self.feat_extractor(wav_data)
+
+        out = torch.reshape(out, (out.shape[0], -1))
+        out = self.out_layer(out)
+        return out #.transpose(1, 2)  # to (batch x seq x dim)
+
+# ----------------------------------------------------------
+from model.tcn import TemporalConvNet
+class TextEncoderTCN(nn.Module):
+    """ based on https://github.com/locuslab/TCN/blob/master/TCN/word_cnn/model.py """
+    def __init__(self, args, n_words, embed_size=300, pre_trained_embedding=None,
+                 kernel_size=2, dropout=0.3, emb_dropout=0.1):
+        super(TextEncoderTCN, self).__init__()
+
+        if pre_trained_embedding is not None:  # use pre-trained embedding (fasttext)
+            assert pre_trained_embedding.shape[0] == n_words
+            assert pre_trained_embedding.shape[1] == embed_size
+            self.embedding = nn.Embedding.from_pretrained(torch.FloatTensor(pre_trained_embedding),
+                                                          freeze=False)
+        else:
+            self.embedding = nn.Embedding(n_words, embed_size)
+
+        num_channels = [args.hidden_size] * args.n_layers
+        self.tcn = TemporalConvNet(embed_size, num_channels, kernel_size, dropout=dropout)
+
+        self.decoder = nn.Linear(num_channels[-1], 32)
+        self.drop = nn.Dropout(emb_dropout)
+        self.emb_dropout = emb_dropout
+        self.init_weights()
+
+    def init_weights(self):
+        self.decoder.bias.data.fill_(0)
+        self.decoder.weight.data.normal_(0, 0.01)
+
+    def forward(self, input):
+        emb = self.drop(self.embedding(input))
+        y = self.tcn(emb.transpose(1, 2)).transpose(1, 2)
+        y = self.decoder(y)
+        return y.contiguous(), 0
+
+
+
+
+
+
+
+# Attention is all you need
+'''
+class Encoder_ATX(nn.Module):
+    def __init__(self,
+                 input_dim,
+                 hid_dim,
+                 n_layers,
+                 n_heads,
+                 pf_dim,
+                 dropout,
+                 device,
+                 max_length=100):
+        super().__init__()
+
+        self.device = device
+
+        self.tok_embedding = nn.Embedding(input_dim, hid_dim)
+        self.pos_embedding = nn.Embedding(max_length, hid_dim)
+
+        self.layers = nn.ModuleList([EncoderLayer(hid_dim,
+                                                  n_heads,
+                                                  pf_dim,
+                                                  dropout,
+                                                  device)
+                                     for _ in range(n_layers)])
+
+        self.dropout = nn.Dropout(dropout)
+
+        self.scale = torch.sqrt(torch.FloatTensor([hid_dim])).to(device)
+
+    def forward(self, src, src_mask):
+        # src = [batch size, src len]
+        # src_mask = [batch size, 1, 1, src len]
+
+        batch_size = src.shape[0]
+        src_len = src.shape[1]
+
+        pos = torch.arange(0, src_len).unsqueeze(0).repeat(batch_size, 1).to(self.device)
+
+        # pos = [batch size, src len]
+
+        src = self.dropout((self.tok_embedding(src) * self.scale) + self.pos_embedding(pos))
+
+        # src = [batch size, src len, hid dim]
+
+        for layer in self.layers:
+            src = layer(src, src_mask)
+
+        # src = [batch size, src len, hid dim]
+
+        return src
+
+
+class EncoderLayer_ATX(nn.Module):
+    def __init__(self,
+                 hid_dim,
+                 n_heads,
+                 pf_dim,
+                 dropout,
+                 device):
+        super().__init__()
+
+        self.self_attn_layer_norm = nn.LayerNorm(hid_dim)
+        self.ff_layer_norm = nn.LayerNorm(hid_dim)
+        self.self_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)
+        self.positionwise_feedforward = PositionwiseFeedforwardLayer(hid_dim,
+                                                                     pf_dim,
+                                                                     dropout)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, src, src_mask):
+        # src = [batch size, src len, hid dim]
+        # src_mask = [batch size, 1, 1, src len]
+
+        # self attention
+        _src, _ = self.self_attention(src, src, src, src_mask)
+
+        # dropout, residual connection and layer norm
+        src = self.self_attn_layer_norm(src + self.dropout(_src))
+
+        # src = [batch size, src len, hid dim]
+
+        # positionwise feedforward
+        _src = self.positionwise_feedforward(src)
+
+        # dropout, residual and layer norm
+        src = self.ff_layer_norm(src + self.dropout(_src))
+
+        # src = [batch size, src len, hid dim]
+
+        return src
+
+
+class MultiHeadAttentionLayer_ATX(nn.Module):
+    def __init__(self, hid_dim, n_heads, dropout, device):
+        super().__init__()
+
+        assert hid_dim % n_heads == 0
+
+        self.hid_dim = hid_dim
+        self.n_heads = n_heads
+        self.head_dim = hid_dim // n_heads
+
+        self.fc_q = nn.Linear(hid_dim, hid_dim)
+        self.fc_k = nn.Linear(hid_dim, hid_dim)
+        self.fc_v = nn.Linear(hid_dim, hid_dim)
+
+        self.fc_o = nn.Linear(hid_dim, hid_dim)
+
+        self.dropout = nn.Dropout(dropout)
+
+        self.scale = torch.sqrt(torch.FloatTensor([self.head_dim])).to(device)
+
+    def forward(self, query, key, value, mask=None):
+        batch_size = query.shape[0]
+
+        # query = [batch size, query len, hid dim]
+        # key = [batch size, key len, hid dim]
+        # value = [batch size, value len, hid dim]
+
+        Q = self.fc_q(query)
+        K = self.fc_k(key)
+        V = self.fc_v(value)
+
+        # Q = [batch size, query len, hid dim]
+        # K = [batch size, key len, hid dim]
+        # V = [batch size, value len, hid dim]
+
+        Q = Q.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)
+        K = K.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)
+        V = V.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)
+
+        # Q = [batch size, n heads, query len, head dim]
+        # K = [batch size, n heads, key len, head dim]
+        # V = [batch size, n heads, value len, head dim]
+
+        energy = torch.matmul(Q, K.permute(0, 1, 3, 2)) / self.scale
+
+        # energy = [batch size, n heads, query len, key len]
+
+        if mask is not None:
+            energy = energy.masked_fill(mask == 0, -1e10)
+
+        attention = torch.softmax(energy, dim=-1)
+
+        # attention = [batch size, n heads, query len, key len]
+
+        x = torch.matmul(self.dropout(attention), V)
+
+        # x = [batch size, n heads, query len, head dim]
+
+        x = x.permute(0, 2, 1, 3).contiguous()
+
+        # x = [batch size, query len, n heads, head dim]
+
+        x = x.view(batch_size, -1, self.hid_dim)
+
+        # x = [batch size, query len, hid dim]
+
+        x = self.fc_o(x)
+
+        # x = [batch size, query len, hid dim]
+
+        return x, attention
+
+
+class PositionwiseFeedforwardLayer_ATX(nn.Module):
+    def __init__(self, hid_dim, pf_dim, dropout):
+        super().__init__()
+
+        self.fc_1 = nn.Linear(hid_dim, pf_dim)
+        self.fc_2 = nn.Linear(pf_dim, hid_dim)
+
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        # x = [batch size, seq len, hid dim]
+
+        x = self.dropout(torch.relu(self.fc_1(x)))
+
+        # x = [batch size, seq len, pf dim]
+
+        x = self.fc_2(x)
+
+        # x = [batch size, seq len, hid dim]
+
+        return x
+
+
+class Decoder_ATX(nn.Module):
+    def __init__(self,
+                 output_dim,
+                 hid_dim,
+                 n_layers,
+                 n_heads,
+                 pf_dim,
+                 dropout,
+                 device,
+                 max_length=100):
+        super().__init__()
+
+        self.device = device
+
+        self.tok_embedding = nn.Embedding(output_dim, hid_dim)
+        self.pos_embedding = nn.Embedding(max_length, hid_dim)
+
+        self.layers = nn.ModuleList([DecoderLayer(hid_dim,
+                                                  n_heads,
+                                                  pf_dim,
+                                                  dropout,
+                                                  device)
+                                     for _ in range(n_layers)])
+
+        self.fc_out = nn.Linear(hid_dim, output_dim)
+
+        self.dropout = nn.Dropout(dropout)
+
+        self.scale = torch.sqrt(torch.FloatTensor([hid_dim])).to(device)
+
+    def forward(self, trg, enc_src, trg_mask, src_mask):
+        # trg = [batch size, trg len]
+        # enc_src = [batch size, src len, hid dim]
+        # trg_mask = [batch size, 1, trg len, trg len]
+        # src_mask = [batch size, 1, 1, src len]
+
+        batch_size = trg.shape[0]
+        trg_len = trg.shape[1]
+
+        pos = torch.arange(0, trg_len).unsqueeze(0).repeat(batch_size, 1).to(self.device)
+
+        # pos = [batch size, trg len]
+
+        trg = self.dropout((self.tok_embedding(trg) * self.scale) + self.pos_embedding(pos))
+
+        # trg = [batch size, trg len, hid dim]
+
+        for layer in self.layers:
+            trg, attention = layer(trg, enc_src, trg_mask, src_mask)
+
+        # trg = [batch size, trg len, hid dim]
+        # attention = [batch size, n heads, trg len, src len]
+
+        output = self.fc_out(trg)
+
+        # output = [batch size, trg len, output dim]
+
+        return output, attention
+
+
+class DecoderLayer_ATX(nn.Module):
+    def __init__(self,
+                 hid_dim,
+                 n_heads,
+                 pf_dim,
+                 dropout,
+                 device):
+        super().__init__()
+
+        self.self_attn_layer_norm = nn.LayerNorm(hid_dim)
+        self.enc_attn_layer_norm = nn.LayerNorm(hid_dim)
+        self.ff_layer_norm = nn.LayerNorm(hid_dim)
+        self.self_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)
+        self.encoder_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)
+        self.positionwise_feedforward = PositionwiseFeedforwardLayer(hid_dim,
+                                                                     pf_dim,
+                                                                     dropout)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, trg, enc_src, trg_mask, src_mask):
+        # trg = [batch size, trg len, hid dim]
+        # enc_src = [batch size, src len, hid dim]
+        # trg_mask = [batch size, 1, trg len, trg len]
+        # src_mask = [batch size, 1, 1, src len]
+
+        # self attention
+        _trg, _ = self.self_attention(trg, trg, trg, trg_mask)
+
+        # dropout, residual connection and layer norm
+        trg = self.self_attn_layer_norm(trg + self.dropout(_trg))
+
+        # trg = [batch size, trg len, hid dim]
+
+        # encoder attention
+        _trg, attention = self.encoder_attention(trg, enc_src, enc_src, src_mask)
+
+        # dropout, residual connection and layer norm
+        trg = self.enc_attn_layer_norm(trg + self.dropout(_trg))
+
+        # trg = [batch size, trg len, hid dim]
+
+        # positionwise feedforward
+        _trg = self.positionwise_feedforward(trg)
+
+        # dropout, residual and layer norm
+        trg = self.ff_layer_norm(trg + self.dropout(_trg))
+
+        # trg = [batch size, trg len, hid dim]
+        # attention = [batch size, n heads, trg len, src len]
+
+        return trg, attention
+
+
+class Seq2Seq_ATX(nn.Module):
+    def __init__(self,
+                 encoder,
+                 decoder,
+                 src_pad_idx,
+                 trg_pad_idx,
+                 device):
+        super().__init__()
+
+        self.encoder = encoder
+        self.decoder = decoder
+        self.src_pad_idx = src_pad_idx
+        self.trg_pad_idx = trg_pad_idx
+        self.device = device
+
+    def make_src_mask(self, src):
+        # src = [batch size, src len]
+
+        src_mask = (src != self.src_pad_idx).unsqueeze(1).unsqueeze(2)
+
+        # src_mask = [batch size, 1, 1, src len]
+
+        return src_mask
+
+    def make_trg_mask(self, trg):
+        # trg = [batch size, trg len]
+
+        trg_pad_mask = (trg != self.trg_pad_idx).unsqueeze(1).unsqueeze(2)
+
+        # trg_pad_mask = [batch size, 1, 1, trg len]
+
+        trg_len = trg.shape[1]
+
+        trg_sub_mask = torch.tril(torch.ones((trg_len, trg_len), device=self.device)).bool()
+
+        # trg_sub_mask = [trg len, trg len]
+
+        trg_mask = trg_pad_mask & trg_sub_mask
+
+        # trg_mask = [batch size, 1, trg len, trg len]
+
+        return trg_mask
+
+    def forward(self, src, trg):
+        # src = [batch size, src len]
+        # trg = [batch size, trg len]
+
+        src_mask = self.make_src_mask(src)
+        trg_mask = self.make_trg_mask(trg)
+
+        # src_mask = [batch size, 1, 1, src len]
+        # trg_mask = [batch size, 1, trg len, trg len]
+
+        enc_src = self.encoder(src, src_mask)
+
+        # enc_src = [batch size, src len, hid dim]
+
+        output, attention = self.decoder(trg, enc_src, trg_mask, src_mask)
+
+        # output = [batch size, trg len, output dim]
+        # attention = [batch size, n heads, trg len, src len]
+
+        return output, attention
+
+
+class DNN(nn.Module):
+    def __init__(self, n_layers, hidden_units, input_dim, output_dim, device):
+        super(DNN, self).__init__()
+        fc = []
+        self.n_layers = n_layers
+        for i in range(n_layers):
+            if i == 0:
+                fc.append(nn.Linear(input_dim, hidden_units).to(device))
+            elif i == n_layers-1:
+                fc.append(nn.Linear(hidden_units, output_dim).to(device))
+            else:
+                fc.append(nn.Linear(hidden_units, hidden_units).to(device))
+        self.fc = fc
+
+    def forward(self, x):
+        for i in range(self.n_layers):
+            if i == self.n_layers-1:
+                 x = F.sigmoid(self.fc[i](x))
+            else:
+                 x = F.relu(self.fc[i](x))
+        return x
+'''
\ No newline at end of file
diff --git a/scripts/model/__pycache__/Autoencoder_VQVAE_model.cpython-37.pyc b/scripts/model/__pycache__/Autoencoder_VQVAE_model.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c8c8e15c6f73ed328e6b3320ab310e79fe4c197b
GIT binary patch
literal 28281
zcmeHwdz2*CU0zpp_4_eBJ@4Jw-QLwo+L710T3HCABkQrdS_vZ$mbA#6ww!ivSI<oM
z&UDY-s@cb!>eyi=<A4|B7zidL>2VTc1Mxor9~@!~*nwahhXBXmf?&lqIKsrSoyYNU
zNSyEc-RkPc^sb&c!RJtO>UPz=x9+`l>)zk@d)+%HbGdW`$0t7itLHxaCy~e(dDA;A
zT-=Y-xz~t9?1*iwN6P#)*NqLcY#KT)x?z>A;5xQpnGqwh5ici#m}EI=N6V>)BDQ76
zo{HG9y46fyjg~WriQ5UpB!ZYMVv=?WF{vOXhnTdTK};ry$s;Cf=Ma+%VkQuiw<i!Y
z5yTV_Q?MryGa1B8BBp3hA!aIwDI#Xto<Yn^5Hp3CS$hsKb3x2BV&?4y#4H3cGl<z^
z??uesAZ8XZi}pUm><eP%5VPMtfS3b8%)D(p9$6|q$6j4tGW_HdRn@4TTd$v~F{*d4
zvw1j25Y!`O!;X|qJNd(r6+^l%RgT&z=`y2i{cyyJlw&A1?WZ4X)!KGlJ+{3396nRz
z%Pb@kdG@~LT3la7_?y5P#pyH=xDnUzB3I3>xr49e;kpv_OoWyfMHuregmE{~O(M<Q
zVM@wPG}FjqGDkCR8@72ZhWL!lC$es?neR^Q;47cE)QZg#vvvdxj4hvCGW~?D+MDey
z*Uz16*bP;$xs7(Kx_(CSt(OXZ%-PzkE5!!-$x5ZsYPgk(pKi1^x7>=;*slATwT5li
zTlyy1s;pP9)Rm+7YQEWO`SC{@t$J1Y$wrXoM<06Z2mI)&+VW@Yc4cL~>biJ-v#P2a
zbr;Y2F}JO1Yra+6va5d9u3v1_>Xll1tK}}meaoq@uPDw9m0^%&AhpeGs(Qs$Rn%u!
z>Ko_kw%us0;=`sYmD+mMaVnJ$M!xl~<Lm8Ob^X;&O;z36+VRb1d!v4(x_w-=okqQ~
zT6f$nlyk1$s;zBQ)%oMK^)?DTerkQoLF-WB@eN!ZKh=Kq<D2z*ZLRW9@W?~;Rvl%c
z1COKd#-{5W-@pJ^KmNd$+eUF543#Gyed2)!D-wTf^NLEL^#z>HG=hkcH{$xw%p3F3
zIPSNn`orj0?*r8MJ$S@$^4(A2bbbs0hCoCW5E^<2xlvVeEqr#P8M_L?;6=J|&)6|K
z7#v*#Lk5HBNwZ^hOwY1S+k6_Q4pBzpX>%pI5E)KkJaaH*F4D13!#hwz)U%o~&qOUz
z)M9z&iitsHReuyiKV1rryX7O7g3(jI;M^T1F-LFymD(#MXRW$fFCA%ZZB)?hn@7$r
z9X-eICFAVK$)%;EC8w?2x?QsCHC3-V_%Q2NO6RVW*6XcRcg;Dn(pII-YVAD7Kv>OC
z3xb@(jaY9ut~8*2VGAFv9+Wp)f*$q5k8*Z2s_P}kt-5uVfq~wCPSr86S|hKF93@OY
z%q9oj(tGQ(M^7Fdm{vy~Uh2OAI|_r@E$IwW8S4U>Inp>*Kc*9PnI-lCs^~xM&jiD%
za;B{=Rh5k?jmnp|7f0jStfN6COG#D4;8K$a{NlJ7YuC?ht@^P|%nuhtgC9>#@Da|@
zrI?!KlRdJ55AVly^Zn^?FrX!xc+_n_5%h^#U>OqZVF?Mm5L<l!H+srs<3`%Bj4Z-}
z`C42RjP2sUlnEYKiV8(joG_sE%OE5mXDv`1lCugY<kmW`vz5w)t?If?k?E-pv*uht
zLu&Q)^-87Rt|ddwBVDn1LMHjv4yk9-$WKU24L9NOa?#|}^0K_i7&IZ0uyzs$=Pt~<
zR}xKtGFdwcz~f)+8gVEms6*tz-)g)=T*@<_7q{ae8*v*X!cO21+{3jPQYL~p!%kl_
zabEyhWv&^)GbA-}+rl$h{0%<Cxbe>*1<z!W&UOtx!?=-Wc2aD$okTe47MoMuY2ae?
zy!DKs?hs_06lBa183j@lyHvXSITKe)mY=~Ot*upCt@^s-XE93qcbW5bg=h_gamQ5+
z(0c;^a~yPMFbR&bv{y4Y#^5*t-#XW*I>4xSb!7#>g{uT+#Co+>e~#aAX+|ZPcZmVP
zVjS|R924>kWX=&#^jwXfYPT+)u>EKaNSuCLNE%6$pE*fly4+S9NVOiTKk@^5hEMTP
zVv(9bu#}LAJGRa_r2ENUU-YIyI8~*wuSFh*Z#{x+<}{|MW$Zzi9taDmIHzw9@i5fK
zBOd3FAYKa>F9PwRT_6{+&&BX<Ca)TThe<DrIMYpa(>AY&Xe&T8Yb>J0{ua@QKs@gL
z8-ZwQF9t!6yuS+9GUp_C9#G5lK}{|4!F>$&Ga$amT=EkDvW+b;es?j7)5DL0T?OwE
zpqeT%;UI%U3~pjD3|oRAWyCv#ipT(DrLDA)juOBIV2k5^1h#I&V;Z*9&A3#zFu0Y0
z&;%lsqIlA;xHlmMmq<On8)pxf4hcP}AtpRtGA^C23uF5ofJ;Q5J}wb2fJ}l*g%Fog
zf=iQv9K@vnu>xGm*qNbcG%k&MM&nW@z_P)57-s+viGM*o0WKwgOGS68Io+KZ!KLDL
zaY=E+FInTDMW(=YU`L(cV}L#V^BVl9I~m;0h*^y}uVciE#vJusd`xBwhnzn_2s*2(
zEoZas)KA!oprF2of$)?s4sq1oc(6CNhr#ADQpSOeF#tBEK#qikv<930_<bI*c?#u^
z0h`+f#VY}pcmOOAw7Q9%$d1uXYS{6jxFW;=fB-MiP$WwU1X*sn3#nbeD5^Z<fxJIP
zsEYqB1Ze>J8wEkneH({{orTk&hJ7h_K(KipM9RJ(L~<aQ)}eVxZMA~SJ~*j+I6t(!
z^+J*9XE||*6NAV!ZKSQNNqo}SG%!s@u<2er_%g7GROS-S9yT3Tj2XhFC?a^s?Cxpc
zME?e5*HcO8-yoPg6)9uF*m3;EL<<*({vqL854ditM(&*lOQODnzs?GRaUzmyb)i6O
zMr~snDmYQH#Sxcqlid_`Cp%Q4rESyAbhF5p?&dgYcZ|IeXi;$A%r_^X4>4`)ny6w$
z*(QW~6*c1c+I=!kpnkJ9>eZGXbzFO?po#7&q-&xp+O3eY_bsT&T7K${JvEu;MJy0j
z+cz$vq^b**nR7w82&tH}?5bP!6D{!g)%6NEB+B3-k0XO?UmQ<Ge+`)&>eMvLK5Z0@
zSz|lLUffL|DIQU{VWXWV5O`)6=tc@BS7f#=P(dh~s8-sEc1>U(R8AeJio$DjS~uz;
z2Jz@<P`o&DN3WS({O(XkYgMyj)JWr0Fi-(rY_zwWLG6gBV%)VlHAU3H9UaiLXl><f
zq3Y9t=H=nOx+G8+K@TL#yFe>3tn~>JLp9EdYHySR4a8toRO$>Ct-0q=KSYuynoueb
zOH?JUHtK^)p_50?_8TSxNF-B8j8VI>0a5J{d;mYL>YMed>!+Xy)sPEyOs%czzH!D+
zIPDd8qk37LMAn{gN7mJ!7<olMfov5h$ovU+t>IMqx;Ni=*w1k+R(fshGo4xsLT>c8
zaOz={0*8wsl*Qk6R)$ePKZcZ;6mn%0kxh1~nv8T!{6?wgUd5={!AOvC741TuDPxHG
z?ikdWW;4+Qu7a<^2<oOdW+22xx@o8(qZm_B;BKZjl9pv6O3&#@m<=Y9TWzhWGpHTH
za@AFj;YxikgGU*>fq_T`>^eVsZev3|fU6#<75<+yz>iC}T+z>t$N`Td!)I|iR9{Do
zMf3^J?X>iZd?tLA>vH&f{6=g(JMg189g?`DkxeEGig7iZ;Sn<WWc7N))XoSwCHbT|
z1N^Fx(<}@d>8FeVf|~PZPgd9LYOA`%HV3Ex-J|sAmw)EHJU(*YndKT8M;;_&q!Uq`
z&YhUXBP5qr3Mh{eTA%^6al{eEJ#xbdkGdM1&4hZ40=oX9h|Ay(dL|*oIViXDwsGE6
zwoQvjz6H$_e-S?+@$Ya8Ui2!-_N3$}204l-xu|}YwiBqbQ!S{iJ1LYrg>%{?S4OqK
z47oEm)tm)YvaT6BEPD?3^H86YZp8zA*aGU?!}_340_8%kJ(uTE!$NoOPULjtE$I#v
z3|_jjc;TCGibxH3Z-JM6(%${<fo7?D5WIg}bW*oV8T*CzPb~Y^S+!L^dj@3tHRwwG
z@GEN)a7bfEg%QHAv(x>ICRd`65;KXP3amYqO@GdAz(@gmO9cc&)t!0^N^4LaKj(mc
zK$WF!6rx)4QxEp6DSj$6toVu3+IFO^FLTuaD@U!}vY`v7;iNn<m;k~dLsHMQ1a2!^
zPF+z_&<d<jI`Gpp=d6qINcA;Z^Mj?WG%mx`bE)1~U2~DY2aS%OQ&4rmoaIzjRn_)W
zRMZKnA1iZYw0hdAs#Rw|*L9AKA`~i4AZ0VCBk(76tF-D|>-%C-`FKeEJf3k#HX>#o
zHl6TK>^s?P+DMvl>H+v`LY-||hQ&LhU`>NMP)te3@<@S>;K2CV*N0|IFgZ+woN}jd
zI@D>%Py}HOhN3~6P{q8EeG`nufLL?+3|IuxKefStsx@dqg=bKV7$#+YW<`++t>6o`
z4x&<F-y&?{q3bkp;DH*M1|IBP4r6KM3W~vW8M0sDKC12k7=Vp{AQfY)HCq6lD+)Fu
z(oJ|V5PYLGjg--J!%eEUx+w@-28`R_#&Exh`xt~TxkvdK^<k8f1;0iGGyx;-#k)Cx
zMjjJhfTxr2pxV}VorF8lEdZ7hE2bSs?qn-zC-98XNuu=6p!B4d3`?K1Qy@%Vz%xZJ
ziFPJksGaRJ2p9B^9pvoJ2w*`q-JR{-Nxme=$l_3G$?lxwMLkG+?NAyaGZ|R((w&T#
z4$4iV+<6c6b<$qi%XsmXD86>4YG8nbyhp(1Y_vIL^{^i1ade6nX~d{h+foW7qvSR=
z>M%joHwQSKHw&{+x?BQV@|Mv$Xu7J5<J?AVS~uX@-mJMc)X=L*7bsJ4SB)c_&G3tj
zGH;;`?#+BD=LJ@Au|YeU4KwwCc`s0ph8;i#FW^7}UNKBsZoBonOPA^;wbd$--ztG}
zO1FX~ELC+ci*fHtdkaRw_SU*x0^LLsC|g~X*4ypPuryZWwo99;jrKSwuU@*e)?P<}
zfk`vW*=K{){g{Q@`vN?#1T5Oo(%_KTF6~Ml>09Bgmbdrp%BNbvusXB4BF9(nTUr<u
zc7?!FMDneRjrt|wXEn&}L*#&Bw#i!|Rd3;yuxeAhdXhl{fuG*4!=4ZL0FsFewZ^0e
z7@TK7a;(}2ma>}oX9wTZ&kPJxC2U=0;2Zn7OKn(AH4?&FnI#bnV94<k06({W87z4C
z*1mDUk83p6n&M$%D*EA@Nkj!DCWrxX*hRADG_W8GG{^!g3J}Gz(V}SyK4hUaP8<0I
z%$`69Na_r&0%OhX{nsPG+Ik9oA-O>HRs?56bRW-jI?QyK!3}iFWIFx~r$ZJ#X+Ugz
zAZaPa)^LUgE9Bt}v7ISlJ1M=B?bOn{1=k`npo45@#!m&_6>SCHvIighB`Y0e!EZz`
z!dsFDg9Opk+$H(8m<vLUm;_Gv(BL<MRHF%A4{i>|HS(Jf?Ga`K7Ng6seQ%=V7r+;s
z#aLn|sJAf?KI|N?goTj7&0g^fV7MSzRaFa~2^G*tEt6+M-b}U6!7oS4s^tuXFY#%i
z8$+PiVSD^^Fko4L6i(J;JOi@Hp(UmVaWF|@s3b9{^q+}gv|S8)cQ+@2ha&(BjnOnR
z5~ImS7@Bi{G2K)@gm}gAf))tRFfziK)L}DNQpDJv=~p<&7myd=L4hIU3#jt-;B|FA
zsq$dZ<E=%uK+U_x4;il6gsQ<aC?dk~iP%kEA<oYUzQ6+3MYU{#hH-?T57}FO{s9Mk
zC-Lq<rP@k_KutgRDuHb58{6t4ZWR?&etwwRDJq<M{tK5H?pmeQZaDSIMfjtPy!~P1
zd^_J>%YFC&n6E{1dvc%`f~je~><P)y*JfqxKsX`fl2I9vL(`EnhKF-Mf}Rw3BbAyB
zyE%LSjdC1beHlB8tAwc8lIY?Ie{z?PQ_W)eI5<f|0no(vcOzGO<A)-d>FOy84<$@Q
zOi%b1L6jFKwj{))G|miSnHm#;EDNPNr)X{{6~e!WgL!v?N&}P;y~Zg|ZzxlJ$eqOV
zoa*ChDfY_8?{Evi={2X|JV&AK2>8ZGWm?)3X%-}B3_8MrYiZZa^X8a+-W+ofQQ14-
zYMb*x{r}pIcW4xdR2JMlh~FzB?V`I6*ZbxA0Ek#ZgvY1c656$h6nS51B~q`F6dr8t
zaS!cIy~^G?=-za_l%Pcmm|Ky`VU&C`es9Jv%e)1@w+8L}OKIb+LH=9uu1AoXkgu~O
zWHkwCBXF^p!|=e|B=u5N1ZP)YBga<)CFyONrjnfcQCb-}bk*A#h=x?C>)&H)BAA~V
z@I#Rjf-_V*<;hS6gyKrndHz6DKyEeZ7Gk!<8mNl)hXmM*)w&;r3!k5%AKyx?A{yLu
za09)npAW8sBK`cJ#Cq|9Z{m!?O{%;woB$8UjZN@<E!QdME?yY8^<yjR?W$Yezx#oW
zhP#0$klAtl36wAtk$vLP${VX!svDID-|&E+M4`et!_zg0_a}Q-m4_aG9Lm8Hclq$)
zT%jwb<7eQN2^pMrJI7D*N8pMKmLfmpn|6B;lhi+E%0FUC3~yh(G9He8l*H++$Q2=<
zW0~M^^5ALm!s!&F1@Jx+r^A?*fQ!Qa6pU#+L5>LA4!9w5J$YEr<BT_EGjWU1mNc`J
zrDNNBc2T9>@J2x5kUE9ap`<I*PK(RpL@AmZfD6Rs=!N2&bc#jy%Au5_K}u{_%9$V~
zzANRef$$w&hF&L8E)9NPt=s59j@EIgA3^*2^11qP-af<N-3V?ZuIB`0dn2Fz$W9TD
zM1Vx}!Repd2X^<_(6co>E5}gBJ&Y*O4Uye7Z_v`pFS8pBJBk_y>%wB#QKKn$4d^1`
z>c>!|>M|I4YfjG_z3W84Q9SYJMxzBO>3{HTNha{N7;$)4oZAPl^PY9M=<4Urh7d+1
zC7j6^xg8gxp$5yb7uH?THN$-tS5PY`kUFv+({PS`9Bj+uJ@tyV)JL_pIa<HG>8EMC
zfzKmkl%=?qLd=aVKfc~xg(ob$py|r29z=D16td_DQ{m^qNA@&cu=jkMZz=pm6f=Ty
zgSlPW{f@&Aoe5u&WSGpWykZd(0@@ThD$poHWjwweUJ$q%#sYzxrt^9i3VB@7pd;oQ
zTtgiXQ%?$tdFmaIO8mBg+3+-ukwXI>uXyoJqD99GjvF`)K5Z^UIte>U&*WwfO4%sR
zNH^~RH=eXQ$xf<6y+#U89Yvi`>_RDfo!=E};0d5Ju=`1OQa$Syp$WphctR)eCisLf
z8BFb<W|TDzrE{V=V;e9f#8&v`5D%c0K?^#t^mXUtDu=6iFYD!yp2ziqmq&Wu%kz2t
zw0<Ky1`QQ8M?>{^iOz&Oi59ic8mPIOGrBdMiPcC)con3573Jbdyg7ZdC*cM#(<yqE
zl$1cd^i9(-(_WG7MTq_b$`|d#0(x3{eUA)E<VDK{a!I=w+xxy&>mA^iP)cXYo1z$W
zPvo-e?xh$+7X^?xyEBdJ#b-?C5c+AKH-$HzVjJ=GrqF9KeBIcS(auaK(V6vTFaf6T
ziFkZw|M|!>rusqf=+i78D)t#ZDP?8*Wj!OBNW|xm`;+)B;P(UlHhL3-<Al8rZC82)
zHs0&zVLJH#*6E<MZ(KJgN@LFJrPGr=;9B}-x3k+xE<`v6uvhnE-fS<P))HIK@+fA&
zfPCMrC;tJhs`w|!uKqp)DlPn&s44_DwaSC^O`T(s>9+m2UjLwppfEe{Vfs%qps`=m
z$xuV$r>TButZu;w>L;K97gy{9%-BbZ5Ac?dq27hSkAV=aVrc^-wiwVOU};X1B(B@g
z9Dr!i<+41{V<jq8tQfgi*IsX;C<&#cz`;8o@Iisn&rhtku@VD3Pzrw5Xz4Z-C^<OU
zZLZfZH{2_dBHado*HNe0K2pSsRi&QBO?kRk-bzD-s>E`p3b$-sqoY+Ud}Tk^%Pd~z
zzTx^|bF@0fHwJvU`{v8E31Yg95+W&jmS@3S91_L^a0${1;Z1;Z(V`p;wF?ZKRL4kO
zgvb*EgOac;D+&n=d0E0DguYoL9|MDe=aZ73j0&DYnNateOgU^G!f(mkUf4~%d!84@
z*(Kx5QfjQFu!}~k9R~815A*5>gHR^zab<M*Q13+WBR%MV&G7Fev&nQiBk?t)g$E!(
zj+fwsq27xpL!=>)@??R|8iMgk8^{!vOY<%)06ayqN+5>jKvHoX1HE}O1)mG}VraCY
zPj3*DlSNdNR`P)|XH^?0f9GSpi~E))w0yxPiq6K*_ty@Q-SsCbT!a#uy_^AWYf&Yx
z;7KeY2s8NcfLWu>*iUY?8u)A*6}8OU0@?aO9v!+k>P!6ma3y{&EKhjupF*DAcask(
zA1{U24u@(17lDtpS>zu$>g|xX?=!u~D+l|Y=P8`ytV3XSJ#)Z11RYOQ(KV#cd<G6B
zzIAC0=JIhSqI~}qXob7RhVsLUm1wRfcpMwf6}sy1mH2+r#Ah%<$#V(lU*}gqAJ%>V
zCPZF%mk_?hoa1u9z$QE;YaQ+QwW!1Z+^}9w$0n|sb5Izdjs)$`Oz9@6W|)r5lQ~GK
zpLWwQ?8KWHF(akz6rfvQN3h}{L2puctmJI+fk~8~hY=_v&>Vpmse!c)>1#At#k!MV
zLXyabg;G)<1bWs7&`#mL=%r9X8dp<tmBAHl(MB@^R%HgPN|+1J&w?pQb?0})QK^&F
z_jr56pu8Ml682{=OKud5mBDi6!M^P99_2KPfNp~=@8nU>9-NlPtGynyJagb~r*US;
z<xW8Chb<g6O?WZF<x+N8!{Pn-N`0>iuAO)zV*JGUdVLcU2*XzHbNJ$1I;8DOK4nwb
zpy74_D+TiW0-jm@)sOzs=RWWK$?KLDwL(q+&Sm-khDAo5tqna4KgQcQe4aM8*40Ot
zAw)oN)4+NPwF)*~EBZgk3?F5{TKuE~4-c44DgrOUVXJ@1)L&$9R<dkB*d<aCf8A<r
zP5lPb-p8~k;0W$oGHv4mEROnDOtAQX`ekPPFz?d?ubT?r)i3nmt!Fb*zr@_eM%moB
zgozI!9UkT|ShM;ZW%mAUs(Nu1LV5d~1ABJ}ntj^dhp=~Zy4{X~Ban6$&3y7Cp$and
zbaaoA2bj@PP&Cug0)QtE2+JF#XdxDfZWqF-*~6Rh=7E2UvY*#HK)UL&DOwwuEx<Uy
zxEjvz5UPMX=N1G#ZKNF+eI(4XVjO_ti#{pg0^5Z8hsgZpudjXaoxl9eFWoz|rj~V)
zyg5IE;1!1#0t$tIJm`g(f_ET9U@l^2i<uYV66ubpyU6NJ0i3YX2*3o`s?3O&%d7yB
zwXE((3&k4aPXa=!Sg0z?OvRK57A(nRzNoe*a2Du+EJiJA<F!4DAk?23+Fj|_8OLGk
zPpJATrW!{4xviBItmh@;sbp3>#Ps-CS<&Ffr3sfbRK+V`o~l={P7cCn1B+k-Ayxe*
zAO1K4qLTWjEFsa$6D)@t#;8wo0Q3zIag9;YxJBnevPRo8y)j2k)o#4vvZfFWiYN^v
z>Lr2!J{-UR1T7BoN9UbkSRgTuLjyeu3n2YErJEXng|vE~n-N&Z3M^y<7W6e_nGhE8
zAuLP)7Qm@MVuOqWqCOeI0%g-aEF?Kh*pdJi00zjN1q5Iyb;$M1Ku-L5{LbQ>n+CM;
zcb+{1kfB^lqK^SLD>2B|oF}jZPb?sf*OWLHm~$`gsU?>DsGaYLelj~5cMsrzy5ymH
zybPa`LB2SUV;0i_5J<%c^2aL)cVBlufa5?9q7YjmAPv9~(gHX-7{XBwtv)0z=haPK
zZUm0<0!QczFaLtzNc|RuPyj{hw>fVAosJmBox|gBM^CZe;X}d$5q#7q8H_<5^(iKt
zWH1an>eGx1amG*f%z6sjJ4Dpy8IWaE&oYqt&B>`~Toq7Kzssx7Fc2_0&8zn@`1c6F
z$P$s%?=c`8ssF$rL?iY4y!`_Pf5<>%P76BROX{<X_+tj2V<3~aKZ6e=?n|7(cjF=g
zya1XQ=|qSpSWb~oz&Z_42dnv_`RE}a&LMNrDp*995Lc#$=6DZV4CSL#(3YF=^I{h^
zkurXYO(TLp(wY}qIfO050Upl12zuBu*3y*?HgSlFr7HqUm+WydVCTyD6X82(eff}7
z6GIIGJI~<am4hA37;7(dNcxvhgDWlb3`jVDAGpE>n?g(kcc228(S<<z>4VMEq(+uM
zM?+sKvMlpiftNIGAc1Kit7Ub47YH{ah%sJ{xg9+>xC??vG5byUy;;}=VSc3FcS8g_
zp64l?FHfXVpWvH~k!Tb-p3`h6hnt8o`$yX~EY1ACQz7e#|Ig5j9xVTJT&sV<;Qb69
zWbktgMyB%z5ce=&VnkWI>s5wT#Y67e36von{RA$c+vVB^19X}vWJioupwEs;!2vO_
z(97xwy35ji8}795)ME%cJ+PLG<@gK#yB!lHK{pBUBDUP1*td!ra6Efo-&qMR8N$Gj
z_W4Oj0B&P*{fhcN+$y2cCz%k3jvWei5eEV*71l(64f5m8M!U_`5@e*gX#v>jkzVps
zbd?uQ`!V(v`LKwQOdm=dv9=dRKYEyVy!5(`*I?(oH0ia^u;2;|3qU*p)&)+ABdiM<
z6!C-?Ikzv-ri67tN;e&{E=f0|e$mA$dq}i7VO`+s4c0`iZAzb_cxMgxAaqUy;{tiq
zq#I!;hV+y$C{*>6MTo#9l16$c#k-SWP%>^2@~iN@TtWlMRPG^}0<v+hCDWM!$rMjR
zGDRv_0h9-}W>7MvOd1!-bZ)F<nqjRkP%<6Q3IyZqYxnhq(D|{<z+Z5feVM@v<p?-V
z)mIsW=zo1c7cAG(<&Cl2Pi{j*6ve2Z`d3)Ne_=pq*O>lCyt*N<H#xeGAzL6O(t#`<
zKzkAnXNCo*vSNHdNHh!4h`Al=Zvw+{xMCT)$?=Lw5>Ks@l&(0;+anA@`qTpk(QW-X
zl8*Pt6cm%wNh_UB&MU^(aE1r+$nlB*yrRT79@IaCl<R}KSdHOb2K^DFKb&w?0rVz=
z7&7yL!V)oLyveTXJQ-YNn%QQKF2+m&*s}!u24JsKv5OO+pVDVxpj2JPWEi6%U#}Rp
zmY*x2)dRRcrE#BcIfDD;jH=sPHTcuhqfFS(Jae5#;Af!rheAR-<6l?STxNb@O~X0z
zVmN@G<JTUP65~*lQ<tf^fTFRzXN2&cxa<EPAMjDOHsk|JyIc01!3q2ZUt@$581pjU
z5xju<&m4fmY=C->aR(6eJXQXZ5kf5gidRB0LlUI*XCzRcV#f@?eqwv??r*#s_$Tpb
z3g;=D&bwY|;0MG5274?3RMjIa0EthsncAhjNDB*qlx}8#1<0xoyW9>Z-pmUNkP{X_
zu7w4dpapt>1t^TL0BK<XP%2pfv~1GLfH@&EP!wh$AK2D2F19oPGcXH?*35w1gBd_f
z$PCQwVg`^(X26Tl0}e`RSlPiCEQE}~1Zv`$@EGoajd)mPjYVIm71n*P@G^7+V;F<I
zV;KWdN)=MRm%tr-odw*GMbK((EZ{uNtimcxz7VnhzcTv`xrDN|+tTP-dH8|HPT!bo
z_zO;sJ5gxBH9(a-hHs#gYmak)l`|hLY2IO!e+C&3wE?dX_dtMqe~+|*dzcmOVXuaF
zw!3#Aj~ov9ksGN8LdA0oN&xJW*jA4Q_G>ksU$}?@ug@=tfUY?O%@>R@ZMtc-4D})v
zX(>)n{@#F3AZGwK35ZVCa+&6p2Yajb1{i=~nfZT1<3<>OKIUW78@=>S^`X9RUJ>or
z8~FUMGa%l-7?eK(%nW?bxTZ~X!D5-%Pbq)6V|SxFoz~(wg}lx}^Mi^1!sRMjz~GxD
z&K}&AgFG0CSIUj6C)@<wR#Qz&Na2ep6)i}kTqtTuT*Y#Sm8o8Oo>sFANDx=8N}V8H
zSubTLAfI6aQS1RjJ0!Odz#g4JY)0&mknbRGQOS#xZXqPuSvL-)Z+57q2|MRyd#!Fx
zdRdf>Qo3*mLj6#lHmAH?6HYy%a;7`+jGYfGkyz0R#VXe2izO0vJzXQ-1PjyfblRQc
z;xYOy3gOP{ds<h8a8E$uB0<iF_C@prq&TF4a6^i0?h#^*_H+t(r@c6{9<LU?f;};0
zUjzq>cl%SkTfs~B-UmI4#UQ+A`)z9lb0z5II2?r1gSn-rCj&x18Q3Ptplw5G*V~PQ
z-Y|FT>G0c3y8VdkHF^w&#Up)dUFo*lN_QOP`nA`cI9dwqn6Kmd+a)n{_H3LYNp8Qo
zUxc^>^<4puX-|N`;+6s(^!E4MQMwJww6M|XF>S^is2_V<(zj?=LC4%9OG`^*HP_oT
zuZA8qcQ5T5fT+*lP5qecV;UGo*I;+8ogc87y2v6n)&cyvu6X+?W~Pp1tcLq66MmP0
zhysFCA^{6r6>khz<!D>2VsWYX_K5Eix7?G>X2p?7l-R$7!u{yw%a;e#*`mUJesT9M
zQlH20yZ{%hp`a#4@E6aM4pF~Ni>Qe-of176cpFb!w4Bb;VoD1tHQPC^y;@|t1(A4!
zdi6l<rRlawqHkq$IfemYdBvC-&hTIgj~8V(^@}6zgvIT7%&0$wI$j=>flO$PHP&tb
z$Ou*1glfEC6RLiaCe&#Xs>+N&ow+RLlir?H-$F~Ti)-q?GqsOteI)Bku7AobQw%=H
z;2{S8k%5?D#7OfjZhK1C-(=Fq8L*GU7OMUe)5qFD-^0%tNVp;P(LF4*gYMn^ZFYlT
z9u<T##y0{PgIZXSc<5diBJf?MuddNFxg!lX(a`tQz2ZwiWq>WHs6xcbI{_oCyc00$
zsStInRv~gL4K9xp<>G(}5%JB8><vuE7r+kcfMO6<&sft8;6`u3kObUtO?3fq6AR%c
zR7--SW8(!pP3eW6u?oN3nqdl7O)5mD#HvY!C!~kKs#!#<*xF)q8c)(Ej8DwKs>y5k
z<u+#mt7cNHn#_+H*?w8$of21>2sUvTs>e%(R?Yc=97X+pfmJgBwax<MjRZG#5MZQ2
zXOZ?(06tg&0YRhz1gWas$CiteO%|=#FD>WQ0f2RC2n1>CEYWHV=OXDtJ1UTeO(D=b
z8I+vE3T|r0(*ppf@$6L?6+^%jAPs(g0ZDze<{>@4|0f~^4)nC>Bfv|f`(I#ruZStL
zk3mcM0l4@kD-Zzq177_hgAX$x9;h!e_!5K9G7#hE4K<(d=VZJcc_YzTOu=+CA3GVH
z#<E+I_z_bkh<#wnv=&XMp@s=PT^4$@-cbJwCB8&+CaKpyV6$-2oH?Z!GoIw%fs~%S
zVjEaJJGhoM+zHFpTe-lM#)^~^95;Spp!E-__SWY1JHiFw-%F^yyL4n397uHoF1gOp
zQh&qSqos@W8aD-6D#2S34#TyQ-U>5({`+Ms>z7!-nOXwx#Di)Vc0m-UL#M3KC4*jE
z-D1-{A0z`Sd>L9T>`%-)I*Ma28ERjH{qhAaNL<9FVitVldDD5)P1>=mTtZ4dTT=E$
zWDy;A)93nMkQeTs4@(FM9XNH-=uWr=C^|q;yG2ZXkR#-xD?BV0!GhSta-cH!z57;w
zW&hJ(dGuF4bl>V<fAH(S|AupK)Qj4-??nG`)9z45C>uT1;b&G_<tOQ_@jr(p{w?sX
zZk_(wG>7#z1Sb6#1`As^^~Agi{My#*{)FE03!c?ML<*k}3);DgD=p5fQiq?BYyF-5
z^b^(ft-4+d`x7YJpOw_$xe9#Eu$2m0roE5E4=Kkc^wE@{N&STQl26M`du7F`yI3E;
zLf@mUP3%Zkhw*L}iRE;ET&Zs}A`uL(a*D37QW-LF|92UQeky#ea@K`+UmgE|fbTmV
zYQMy3bPLMGA~UdO7XGj_Oh8v}z*3+WHkOM)$&6+8u<7;Z^h12n-LQKUg=KJ_!s&bg
zV-sT{qLvxDkcc1^VavE3`~e1F0QK+-2!`YWi441$!H&kI${1@LGw}H24y3&ls|CqP
zmcGPb{)}QN4M!xMe_~h`633Z%#<={)fYv0v6tHipc{=ip@s>}(*A$EuV3m-Ky|1|F
zRbni7pXxZ|g>S2cWtdsa1jKjYHV}BWa<?OTs!nj(2x=?<j3*IS<Q63W*7W7qyYyiz
zTs5!^<Q^#ZWM?ESu8llx0+0k|J-9v5)rgRbe+&Qy#D-K2$t}pWV%`(E@U=I=!H5dK
z?i@~dpkhf)y19ThC$AaZJ$M4rs+a5Tg%T1aQ7wqN=yWtAOKo_!PgWjBq(w7q(N)gF
zJvP*o-Tf#r4|vRfcSOn=tZ4$z4!-39ye0D&yh68x`k?rv42-+Mr`Qh%k(%lr5^1+6
z?J0Ogv<(Yonm0**=dN*C({yJB?7(4fTEDX_R=HXC#g2iJ<46O;%3d2yyYP1I2GzU;
zOoDvZ4i*RFPWDZQ+&t&r+C1XTp{|t3?Q<S{2c;ZrK(i3(F6ka^-iDq!U{fAHU>D}`
zKUTnW&3p6RS9$YX5)4E9QE&Dt4fP8i{<^Q;iA+Otfv1n{U|4td;O&lkd+-(ZcyuYd
z9n8yKjJw6lw)FSSG6J3egR|H@EIq-IbOPg)zknZ%7qfW>r1ic1?~ajByk?^H;YdJ>
z<y&jjFZ%{df4(6-@EUIsGXkTvvrk6wzTj@Zx4(M=a{|i+(YgcP0WS(21HP2(+EsFI
z?cRx0-s8J=4(faFfUco)$SZYj@(zjH++pva{NC){j5qu)@381CZt?bXZuN?K9CeO(
zw|F;sw|Yl}&6-+PcVfB^@m1Q_`F=z<xG{n_K?l5+_E8}5+qF#caYTxr{wik5TaskY
z-)&w1D>q>OG=1~f?Y?ojA0_sPZ>8_yf2H91;Cm0oPz-8?+`4^q7tIQ|kYo4A|2T2(
z#YOMIo?CX>t~<3lHmkvY5~go#igR`VSkHc}cX8iRlv~rc96t&#Y6R8Gy#iv`>85^p
z>9y+rpcU%%48D)SBM5wpL|S{vi6Waofei>Z*Bh?pI)%9u!RwQJ=pzh1$lw42Kxjn$
zJg>;Ist++Z%>c7gyS`y}zRfK^23){@Px%y+PBM5D0b04W?x!DVw^p@3tKP_-;G#Y)
z>2~9ydVp6J|91$$YI74kp)d%8y<^TYZ-Tc!$!sxf#pBle+(y+o55p*wG5`tqf7e^0
zd-tt|Ti?*Duws13<wIHNpI+ve*0l8XA@5-xy-2IYMOsTo1rL4m^6h>mz%}W_6iejy
zK+=hFP-<;$t99NfXXvwne?1^K$;&77rEl2fgsq{|HyUO9>m2--olOGy=9TiKZkIe1
zz-yvmU#@Jpm*MTQ*1ja^<y`2U&bKaSnbM0INMr-b(?Q4d(#2m!Wzle~aJMh30<Ov_
zX{$KiSnN;rS9~Wd)_1Xe+?6ss=rNp5guB&oz?Rc`byfv_`sLm>c2v))zsqMWq~2a;
zctXNEMm-*c`TrcUYCrrWDwn<P%5kh)!zV9iq{ZM#a5FO5)L}fwO)H#DUVuNC#owX{
z{sC_C<Rj*-d!r?&fGp#+P!tuh+%|6<=K3vT9#@5UA)1BZdeJDsO%nIfqbz|NyrGPO
zaR*j-Asz}Qs*wsP5v#tee2lBguw-{o1+Mv!U``Ja6moyUhESCSQU*4JEXYpe$L|K%
zz^vfvJ#68Q5V9oVk1PK;ihjkZK3hT}{3a0SM<b6#eh|N6K-jJ4TuSR1FMJ-hmBkpU
z+UkF!0Ru$f*LbTb0VZQbO=1M})@mQ(fF7~5#t~OzXKPC<|C!6YvAu5>VH@HgvGy%F
z;x6hZDiyn3!+&NG#+ctwvFsapnEW*|{TbOr)}6F0GryRuDaK-ND!yeoul2K4>})76
zOS%icT~XS$T4FFMPxWq(t!%Y|f1ibwAK|t3^bs;i8;T!)gZ!%*y^X~ZVq}vEy(waa
zw{K(cP6qE~AkP1EdJ)@|=m143l!gv;r*!Nd<nxpF=>KNJe-5YjOosLle^_&vw9Mj>
f;{C-l#e>DSMT*6}2%X~h7wzIDT$#o5#Yg@xJ069_

literal 0
HcmV?d00001

diff --git a/scripts/model/__pycache__/Autoencoder_model.cpython-37.pyc b/scripts/model/__pycache__/Autoencoder_model.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..86e0344019b7513176feac2eed79d77967ec1ff8
GIT binary patch
literal 9284
zcmbtaTaP41R?b^xWnH_cr{_Ml%f<^+raiL@EASfQ%Z#UOkhfW8#vX6m-gI_lR(I7@
zXVye!&85g@S&!{~SPk+M60j20k4QWt@gERBAdnD(hn3-lM++X{`y#S#J#FtwWGka0
zBO)UrBhLBGcTV)xMx$!r`iuAeImmzCF#eqn)5}5Tb=>LCk%UHQ4vcO7S_5<6+O|wx
zX74-OPLVI|JC<P@`|ftRDDk$v(B7`RWrR*x`qT(Zk<+g}vbSp}al<l7%0)>XC0<xT
zNu?-hprjhsP*N*OnkcD<4U{yBl9kZBX|$VPVur?6+mzmWf#?N0gJ`qEqUps?;gw!O
z5*gcOXlz@d_fw;5s)<#$?XaRIWNte@H5_BRgl?;{dOhwWVI*#BZGCCt)nBj^!}!<N
zwmPoP8vHHewsEI@B$<(!x$(%FSch1P9C_c)Eu>CvBQ50)Qa3A4Jk(i-tf^$>eidyj
zw&=T|8Cs7^D6fTlqn<VT&B@9k*6@xax*>b4hXy8C+Pc-YWH}VcFd1dCvC|8CBI;zl
zBn}3ff?wTUm8EnvjD+9>rRV#-xR?39toGvJDD%_aIFhx!UKmEPe(+*{5Il@TD)?#A
zieu@%(~F}(NUvAaN&BrE?@D`DjO3{>@w<Z{%kcg%5W#+w;ayqE64BX{PG=MbvK~hF
zdY#DcB%?TMyV6OcL03@nM2$(Ei5l(HP(*$v0*n{>(f&>phP`+f8(Z=H&LBur-~U_V
zzdye+NIJpb#k3=Wth0Az*iZJOhr#%YNYY;9??!1hLO(lE+}Ya?#GNahL4ppiTpNs1
z%nm(X*+=fmwdBQ{!zk+P`EM1kycNX}`oscWLFc_;mR{M101U3YG0GBjM`7?)$>rfg
zQNg@dai{A@46|vv`p;^bXDt`c<CAj%x;$O_4tIiA2{*s`8t(MZkzn%;v5M5x!jsve
zl{wgOvtN1yh0BeJo12H`0mNcrLRcU^A6f^_ft5R<6<VL+)+zcZe`a;<bH;KF%h`f8
zXN&^}BYch#?A+;>atou_7{$r0t_AUOf}cU?tF7XCS#58tkeldk`tq!bEvA~+O1iCd
zFBnFxOYvym$AS+p-D+>_aHo`nUAo$CZ?w`xWKq}(qmGDz6k9TQ*xGs68btAKwwGS&
zCZaVAI(NAHvtgEIP_$g0#GsdEY68*S5w<HT`a9Z#miOgXsTaLq&`Pr)i`WN5eEyz@
zAX4#>PezLt6c2ioVp*nNeQV?D#)3j!db>UU0xlH9IBV$!YA}ukfVtGW99`Cxy3ZEZ
z07J~*mZu7F@;4K4KM*058G~<)FCH!Dv^I)C+MZa0aETL0<l1q%8b&*#U0E7JGcv#h
zH&3ka6>4X@Bu?|qDPUmZrK_he*Jr|jnP}LELRl`>M4V$EDml*{%J?A8`e{69m8rXC
z)pX1{(pBpv<W|k`+JedyFSKoiWr7L=l-~k`0LWrs4S~$3LXHN^*M0x)C>ZD(rJmw>
zj@&5FqZ17VzCWK=+Z1O}FE~8~ko@YDxapbArYh;+VRlV5H{C$d)Q%`C0?7W<gB7tu
z=Tl=F#x8X6TT-^fg)Wt)^G24%9joYzJyG}hn|6^LXErit0yEIJLvtO5M41W~W#!D9
zREPtIGzHbr%4(B3+Epiw&}Qic1Big<ezU&<-L^vKu`<`nWPHgf?Bd4tU#|(8H5d_R
zKMG=Lr&-uu6&}h&n+Xw4V|0VLCKBlY>|<H^!4&MpNmVL>IF#l+^b}+O{`9WMkkY(3
zVUPu~94BHw82H#s%9H994YKgwbQfllqK175jZ#_$4Xo?tnt9qBH@J#VLMbgUrQ;gz
z^gSfGH34H$P|2KyoRgabO+w_MJ+WwI53K{h-Ykzro7g!@P>zKLB)VvAKei_LJtWg`
zg8DI#cpRL-klpJgqjXWz6|7`?5mAqpWMcs}w9S2CX24UR$lG((FQ5;Yrc6+wVao_s
zRL&r&E0X<Ip-qcJ5ug^k*4Q?~I%1Tv=QM*Y8pPdRv<PrlH*U=*rUXb+4O7}-Z@*m@
zzl{x$u84+FkjV<nd7KWDGy=4A5)nyrQ<l@Do9zb=WMvemfa+)*;XNf6&+_G!BOfZu
zXzT;><w~~KOZ~Yuk>=a7L5cLIGo7oq)@vfb>$tTrRjL4+(}0}1Ij$>VDnMq*EK>uN
zENvr`(byUj3%@ow+b(415b~g;%bvhMD@h`EE5Sfp{c<0y_Z+3pnp7w`Fto;`3d3qc
zvh3WR)TZ*ZC0@V?8s3yD)(dsYf_P87h&pL^f=qmmt-j5KdLh2c<QXPl?P#*IzmGL7
z7E%MY;6jR5S?#K&d8prC24(Rw8vFovN;+nk7qKq9$5pj9YQwNI-$I@Ebtb>TM8N{r
zRlP3As^_adRc8gNM3LXcof7Okv+jAT3RyaMoLvN5UMW9t3aepF0V{!&^t>E;3RvqP
z>eWxp1!!%^(^rGNFo=T@=lrJDEm%~ws=xTFzvT4~uWfF1IBi}uYeWnMzTd^N9|2#o
zZcrkm4$y#{-9_r=Wu)cYLyEg!7B{lWbXQSU!&5zX6(~1A*_xqw#}Z*khTn``#pRnQ
zhtGoY&$HFsenbF2p<1jJE!NQUn)o?=0T8rG3^I05LC+^~ujds?5OSyRYi5=HY2cId
z*gWKn&*1qiNFfo-4NH$X|8p4cJjeUO6u>Z)oPTf@Bb=LDI5gfgKB^v6F|VqB@$Of*
z4K)J3TN#?C)O?@Lp6RzH&%*a`?>Ok=+p3SJmG@EJlFltLif(O!pO?V|20<sn=5wbt
zAUvln$=fWLSrS>(;H9{rqW-c{crgA@o(X#(-tZxOz=w#^C<Yw^4#`FeG`TZn--6~y
zR<2Jy1X-Cm3$pyC_J_1baVA(`?wuqK!Sd(~$d$zk0EjktO5@6S^hW@!>P^KX$qTjZ
zt6e3H-GU|iPAHul(L3+TDqWU=a*za_0nM|{F81E-J%BrNKkDu7WoSP&oT+RG^beOU
z^>;-O$_iFSvnb8jXjlb(3vmibyQYbl6Nue(!Fzg#4U`M@ede~9{1Fm)LeEN*MD5&_
z)ZkMMT{gHs#5?I#B!<<r3r~jr4DwC0Uay*-<vK)9{KC7bT8`=P$y{~Tffz2xTeZ|g
z4M^NG$7|%=a1-lzk&I9c71)v(D)9v17UF5r`|+*2_1g$HzyMoYn;;O}CGi2150SJj
zZ8<(>!7{wbT367x_}J0-uTihaoxv$S_cc>oc%ef|!HcU)u~xHuA9IHaEqQ8qw#6$j
z5+E6e=A^_~YtjHlW`krHlX6~yAu!{0)EuohGf#Y+RYJUt@CdjXo`-lY<+grC|26UV
z=%o%eN3V{C#LeAF1IDBY5Kso`pq!VdPY2~}WwHu$Rqk4$i`HK3g=M^B9(d^epU}IP
zd$Zn8gcX3&zu}#=+`~LkH#w<HR|TW4PoG$mc>sdu()-@zlxnN%URbK5@$?E$wR%v?
zt3|(6^m{t5qQ`1p&1<>awXwg|z=TN5xJW^J_7f_t7UX5lqieKNsvM@yNC<#NE9>n?
zaId1_0`vJm(T3K87HG;xN5`OB<{!u8j{0lgz~Vm8EDj?~N75ZC<$GepBOGV>!;X@<
znSc2KKPvqdhq%|Hj~F8Ovf#SDd<@|$7-_*BXKS;SWYNp5`%z1b;ue`#3)rca6%?@*
z=v7vZ>%(LO2RIoG!WNJdRlu@Hv<68sob^U#nI)~ENHCui{Y9<&d&vMD7Vi42<(v?G
z`M<cvPZd_(FUZ<PYf&V|ttYCFtgT{MTjTRjv=eb5R-5fJb2nLBgtBWS{bS^$bFUZO
zSF~1Jx2IT5w9*{xUFH-fU1#olOcZv%$=ppOvO10soPwQz3EN~T(N>W+@ey+Ey2keU
z;%CX)f=p^AS6kRn*|?t|1fUHc3|gHSS6G=;mSO9%=mGfm>=UJVSGwBxYI3-YmC9BA
z8%!pB9f`3*h<DBNX5Cr`)Yk#_b(n%x_+fQ>&2kj{*Gn)6Wq9v^|GHIW>a5x*v&K(<
z9lq)yK-<xK{0A6nQ#tj=v&;bXYG&WWGRZLB$DNY5dnP=_H#|qMw1azg9nUi>if7Vs
zC(qQ5yMxFx8RsI;tjS6d9!vzt<kU9sm)zOLosvl(A(JFsa2xuXPLimVGN3R@%D52(
z#;<`|vk!8IXr1|(1QT98(wee)_7*}oKSIyHg3-SPN#Y`in291?bAF(x0arz?f?dI#
z3Pg+mn-4hD?s$Dh#6+@#z_O;&vIdV!1!WnHE4&NNM^+1QWd}3faXh>DPqa@NY|-w#
z2T}1LQu@yVj~K7bR`*GoK)OJxa1D3bL_z~hlE8BYm<_~)EO8g*g7ShHC|B}`e6#8+
z+$no9%<-xDfQv?-hMpHagqhK&o0Q|&;EpGnVB~ImkuwlyGxHNOv-*tioAe72xu;<!
z5pcPkF!~PWXKQB!6QeVPHRNi^<{N1m37WR+LL_1VGx#ZmLoCg)i0~-7>Xl_+6T2)d
z!f^L{*`6OKy)^RgA*_1j>*=oj5AMRz!v;WK*R1i0g;gk2)B1laWQ>9SgM)Da5ld@$
zwvxytb)?4dN`D8*)a^SqnpQ2&W=i^qXxmkBwK5jACQmMg=3z*F3wLL#>HM8}dJNrU
z#-l=RGVB=kjf#ozI&L>a#7LdHAizdHAJ5ifDl8QfDv35VDkXmTT*og0ZlFy+0&z3^
z20S==i;ah$pr)y6{zcWSWUG@Cke7-Y$3ZY}A+NlxwO`ue<S%WpUW~tT+lbeIfz%Do
zXXS@RY`t&b{Zp8i_D@eMt#PR}+#zuW351hwgZM)v@afKI98nO$oe~L?`FYI{kFs2d
zqlOzwt;G3Jm&ZV<tl>lmz7az|sr2}xkVe_VCGYEA5J~&JH?AwMu5h&Q=yyB53QJat
z2Luacv&a|NByMm_D@<Tdl+G)h%sJIDlH;++g&?R{g8|&JVA`4pKNF8Us06(m;*mCW
zG<U1!x;Aar_~J3b_9R*@IW+>)P-k}9wNXbq%JT+Rz_qdOuHDu(4i9~nYBq|R(i1hC
zMUDGJ&BukKWN%>)%d+*PA38-4?T?CwT&tNcC4S7uKV$L}B;Vv(H56zf?50AJFK|tH
z&R_so;0a>YruX>S<5q3y?OQXAr8rV+71A>vq@r&?)(Zw)1;){E*o7QiBXJ2@3WQz9
zQPxeM^UbM?sMsFdL~<}`M-PUw%IGvsC_um3uBKYn{z$rmWEbaFIIZN7qu9{xhv^aQ
zluek+sn?E(_DlR$6)&@!$`7tv<L8e1#^u*$5k`gyXr$FQ{v$erO9&Z(3uPTJ(g<>Z
zF9ZV3O0CR7?2$CVg7b)oBJys(j5s!2t9}KoQHxj<EL3qM2+fDf_*GUH{R}9@Cv}8a
z3W+{$zxkPU&QLBSP>${+PnZ$)w%|0vvFx7+O++xAu8M`D?=?oY;9we)lPU+20=E#+
zFwB`Q;;EdosN@yCuiw_=gbo~Fx~fa#<nBQ=Ud^AF4V*#D1~aR{CB(tUD>$k+s6W`r
z5a~jH&LayNa^s+Z5rNlq8$p?j!_tTLLG!>pSjn5f>c%TZju!nhcMQ0yd$7$7_J~+o
z6a5<fvtO9$Wqj$myoNc~zA*1zzRguZPW&Al>S8bQ>f&1QTf4C3Usul~Q-!Lft$9p*
z?D44kGO=w38{R&zor6!1*V0*1eu2U&?O!MoL&_$tERn80IoC@1r+jPfdN3#<tEBn8
z-4RY2WEp`Zb#|u$!*fOc8Q-He%M#RJHxhrs0>ZEek>J3G$f-{EHCC%Ql*)=<W2;wL
z-JIe^arW0N5V-J}#c#3^73y9f#E1{;(-z&HnEBSSF)dYRhq4ynNKSW|N;8wQZndK#
z&4uH-c|89bHIN+2uz7e`x1O_CUDu?WvTCm)zRx(7s$mHbebsg>;JfWOa9{A}S*V2z
zgXk5?tFQpX{_%z5w0L?7Y2E@4{}#qP8VUcDWonh5XYLY{8T*==AQcXOfT|MbPW4AP
z{CTfgtqGQ57YJ9|P4$lxmkHa92@2X0>FL5aRu*#m<n-}!cN7=@?jfu5y!bu5B)-Fh
z;v(Jes{>MXo~gr}bON-^=GgPe>Kvyky;t<H>Z_Qpky1i1U;M3^p66I=m)4$L`>}Cy
H?ZW>7M^nb$

literal 0
HcmV?d00001

diff --git a/scripts/model/__pycache__/DAE_model.cpython-37.pyc b/scripts/model/__pycache__/DAE_model.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3a1a35d4f563dcb96165c33e28184918e98bf36a
GIT binary patch
literal 8951
zcmds6ON=DRS<ZJ=Wj(sOy81Cav$L~fdvO^&?H2~jddIV~)3a;O_CBV)n8J``S7ujN
zO}!>Et7oRjUIVjujip5#vBjYpS(cEHB@SGSee(g|kT@Wc10Y*Mb3)t}4#4+CW@UBl
z^fqEG%Mw}2zasv4WMpLg-~YuQ&y~xzg6pl;clW|;it;07W|x7)=Wqw#h42(l?I}(E
zYdy8EH8nNS>3ySVq-nEnXo{-zt!6IO<ePc4(0hfx-L(6~W>HmcDxTq)Zz`VY8{N`F
zy;+9F@^aARQcVS#yjOsxkZP8ovArTR#Z*&;rsS2ODW{rcXe!<kG)t+b=Bc-pM)eWK
zuk19`IRCmUI__T2Z)kDxx*PiKq3E>SUcB*|-*fN#-pd33g<*K2q{XJQOHUKQ(@6MA
zQ}vXl_Lj1*da9?rnQ!WN4ZfP%G~QASrD@{fxw!cJwHwZkAKn{^JFQt4{I_uPKllrX
zPzlvYd8kdbW6WTvd+I|K|Hjnxv}2SwR-}3Q9Zh`3Q;^F^iMhS4KyN@lLEQpwJ5oLK
zIBAoc={z%1Q(a#6X(fHK{j5%;M%uB4T=7^r*7%I9M4Pu}i$-o|lJ5*U;U&CU`<vh3
z^~0-={u|e$|DyOCyiog-^7^~0jZ$m|<B>0f33Z%z95!{F*c^$@AdI!aAkIB6hNIy)
zjC0;B;SX9v&lk}6Ny5JE-yQpdu!HYq-RcZ{SH#9^{;e-I^w<de-hOQNhhb+paJ)`G
zwtM)zgOF)__=4lKdTtOn&Yvs)`Nmdnh>!l!pe5X}b+9$+4*UKCcd{jhLC1I6eh`j@
z@9bd&2YpxE*=qHM1K-=a-Wvy@k1@Bm`bceEAAa=q$oE?Z&ds#RO@H8{PmEy;op(lI
zu+_&)^tQMJWa`T3fuO||a0izm6t$w}wUU-sE&L2Muh#J^sfK2#wpz*=`lNnh0bQBR
z-gdT>xQuA3*Kr4b1QDrI<ye`jxUray^hoLIo`NOvkw_Cy<1?%K8kP-AgG&LQRy~4l
zp^XtKQ!6b|rg~O~edf@oiSm4+<s!k7!}3Ben)oXEH)s9J#iH#z(lDxq9qX;pIJP`L
zbXy0pIT-gb8A~XN&2T87$_4nq{=^p*sg%<$s!SKggS4z+i8?jv{n&2%p_6>(rg{B^
z8`r-O8~2^TD7F%%*g(ZZoMV&R{!rX=h4%}{1Q#I`t*Y5_&JD{}nZ}GmmN55uZL)H5
z-qW@Xb-Q744G-MDk2P#N{I263(<R5bJ9c|XiTuifKoOVI#n<wCJ;#}ks-cQ?lnV|{
z+D0CUfS)q2mOdagEo8EbSivj!7{saYQ<%XIurlZ60Ys(*A`3QK1p2+|YzvILMMLDp
zrIVr1C)Bf02%i}Xf(Rjz1K7~b<HIZ6oTr^ry7qB`Rz=hxT9rI3JEt6BAvqO~a>+BX
zY)sF(V~97xDq2<|piFluG9R*hc~A>$bL~U6UzWYBK)WJ-5Z33Vvo;vjYPdFC_pCOS
zv3&9d7quMi9JS76qdm7U+RbBP>G_Dyx`5s<{IM#oq;2u0&_1z{ySR{ha)yOPiG`S5
zEEs`VRJPlkT}AoZtL^XH{Fh1nJO6ey)xZue2^zonc?v>-C{mOtmMLl!(ku-28&~eT
zhaF!ygQ4iVJtwyA`JMJb7#E)-dhI~_aWBsA!KPRPMs%k$at`mt<-@y9YuFzFy9Qwp
zFCpFUgnfcyYZ!!a2?Kt`ec<+;8!ugp^Qhf(9{_N<Or$=}wF4&@?qdium2>m<%iFtg
z;T4zb2w*5^N3pf*4}zhH3%z0634pD>G+R;Tt4L$k4_!Ylz781NZ#jM69mIB;2|^G3
zCj}|gr#3ED*?f)S8H!Ip#3t&v4`!w#EnM0UXY%v7gU>)nP}L2!YS#6VMqo{#)q;^*
zYRO166}7G#dR42dHne#FaYdah<5i`RWJtT)qDk^vKw27M2$Y5URnIHHPy;zFGLSMO
zU3?hEVnk+LS*MEz)}~k}Hqox3C-bKJ_eZfU+<`Z6dYwD|4BWIna1wI~x&mwzZf6kd
zoG0BM1#|q6z!6u3Kbn1w&cuHM-3OcuMX%t!41n2WJ$t9cx^o}G)kjc}KE#qhfHzA#
z1K(<vnt1|cYGgjagio+iKa=xR_?cej9q32k9ps!SpeC3R_z4TzVgw3;lm)M$bW6e4
zp)0fImyM&`QU0iKWJmcUb+Cq6t&jrE{xt>V0G{bmG6LwU@IQL$e|~^@*CTsh$E+*e
zW$|N_6INF6v|^}q>(f=#D9YLZFYyj?xu_7Ck+BcHmgLx4elKHZR~mHk)z;)!6Hj8I
z<f@idD!35D1&T`)a;binsb?u(ptwr$af)Arh%GP^m`h?KPf`6#6xS$zl@-cfCkRDn
zZ%ir?#1@U*?KMi$^$F63X5q!bzAr#geDNVRF{Dv((HvsYsFOI?8-niu2MTaXU`-R>
zi_2hPqn>}i6F!h7l4F6-1Yl^=Y8O4GbYFjn-UFJRV%gF|+FHq=r-YxxZNQjns->+#
zS2Dp_XkaiTORZ>=YPRq)_qj_CniD^V{=|nV8Wa)-<y6sQijPoaW0@Jdv~$Adi?{<C
zBmCs2?4pFy(>>(}QR!4qse8to$_bioich0zX6dwe+5=r~un)0F&*``v*aD;xnsAI@
zSiDo(Y=RppU9A07LpNFI+TG%`<Z-nd=dp=O{*SSZlJ2K0*PFg0_+F3bL>iUEQE*^#
zg5rwf_xF6y!xxhrcgX>Vn(KnZiO_$<_6Q8NmyqPbjae(um8=JGh4l>24c$0D9(3@9
z`c7P>%Q)#K8b-pmDvOVb%UMSpiufc4obs7PQ2stj0{Q@gGJsdBX*KntCV^JEnG9%K
zq=~%f{OBVk8Xcx+1cpfrAk%#8f8PLVXh-^y0qj+$nw(uCf(45=BFocP<fhZJ-qxi?
z>6+b{Hutu+jx8$s&PDu^D4FKD1*$6>z#1=ql#lY_Ps4&&h^(&t5MK;wJIbYLrCW@0
zAYSMP-<SAbz%$AbzPP8JBMMa_wY%gM-_}Whrd7}(5Vd0+{qULK(tIXHYa_oLVFL#`
zrA=$VR6AUm)+2km3Zg=WT}<+LM`(~QMJ2Rei;DC!VEQUsZzwE99{qLKLAg|p?Wm02
zH*gyfQ|F?xS6YYPs^c!we^rQ^d}>qygEv4K%R349aY;%96uU_6V)LwXi<P_M`y;u(
z5)Sm+aT(^m5AKZ(-Uxd+5)1;F3y=!12?O&#?6my$Uw{Aa|KXpafBbA?Gr?1ub8LX)
z+!J49QV+&`@#{z>OVsMSVSn6<)lf1eK-Gae@?*;t?S5w<h@Y{EO~Wt{ud@-6AkG8N
zLhutO&IvzhV7G>V!S;AK4q|mX&i4aY2rx3i?k}_HEegVLtOH0JOUcr=<m&|DWfmB+
z$*ZjR8uRvv7cOLPOPaDbZpYOb!eUB1yn7&CVWTphoNQ0oTeD2+C3maL#s&AV?To~5
zFMt_mfIY{<WW(?@I{G2*fZG}DWYh~<CHEY5GjRE8Nw1rmYDKTAHQZHTF>dveS~Zje
zl@rUGp|W)<9`gxjK{0{pd$?y?mQsSD&}7$)IW0s?JWIw*5YBkBfS8F42AzQ7ocI*l
z|LeaW{OD_c_0Er<ZE^MS;y-VR$=^oer;o{C85JPI5*Y@z7qonHx{Q2{*iUqqTMoHF
z0lL>2x{+I+Ra&*PmU0M*>C$%MWdN{?ScHruI03wJV?BX+zI5UTX#~kqmYV|Nx%f4T
zFHvNMA)3to1_c*yyd?ZKtPN{^Z@dp~kQi5&<-bBfVjNrY`S;>nwwrJyp~jXL+_Hl}
z$BcacCfocL1+iUx4kFIY>YR%7|A8t2!C%o%BE6;8)HTphOP#C&5wb1E<G5c%(+u}Z
z@CLxCQ}KSL|No2oSY0dFjD8FM-1e^4;lKF5!8rkoX1s-%k5yKe8r18qk>jQ<)=|cE
z1@}5@o=3hus}HY39|sN&9};fB8z|jP+$WwBmDozI@68i6>WPozjtBKk#|qC+FTig+
zk$R{yv=^gt{iHWSS?Z0R%)C(rqrN0Z&eT&;<z3!rDf31M5S;Cer1ufOjl}~7B0Z6C
zkr8`|hW=pgb1u()j__DZTAR<*+_LtW<&Hhsn;$UMp`f!7(xft1BkoYwqv%7#dAVN$
zMH9b6-B&22rT#8c(xRm~9x*$h7*brIpdHVhkQgD`s3xTUE{o`agrLYAkO-I!DaI6s
z6w=A4gIM$KiF;Jsr+7di$(a=An0*?eqQY*84rh6hJh=X*UePbZlU&v=n246avt&M{
zzF^rizoLo>dIh4KM)&#Xv)l^-@g3ZWd(kR&=@<Alv+Mo2mtRKv)7{I@kn&6SFr)k&
z;ZNcV&cJZ-d+7Ugrv56|fn@3n_$|W+32Mve&*QkwHW_aJ33`D=|IBb3+&mxUia0s~
z=LK`0+DW{+TNHmb%MG@{j7w3zTTXd9LX?uXFGU4;?4oq5;P1;Re}}j4u6TCJ-|J`c
z_f^T?m%-oH$lv)4-0LZV){fP61=v``KCmdCk9f&+6Y&W38Q^o*kX)T8UYwps`(noE
zFJu^cKEu!w=J^SUp-i2NN?vh+(U<WIo8ySeDTbEbi_t%Zp=rGGpRba*gA8{<P<Y9$
z=U962|AwWsR?zXl?IQ*uexEr>;@$f^3Ig3t;8li0CB{nJeJ`LDUu6ddHU|e1DB}Wx
zX^5mbZjeBzMAU1j1gom!PyFu?itR81FByh5wJ+Q{8$0)CGap0m8Fs>Uc$Q=9YxoBT
z2U6Co4Fnl+#$!sv#HoX#PS!HSOsr~-nSa1Bzdy_*aJ<4Hajx@Pr7Eadz)aF-Uhnr5
zK?ZTpJkI2oUF7B`k27TmeRkBCOOG1mp=26d2eXl^Lmp;g?J7=@`+29d5PUB$<SGlf
zr8Jf@sV0H>`T6TE+$oVtyiH+KoDzMHja_$ekVN;$N#}9wcSrXa*h4T^F0WPAT%+Pt
zcKl6#mJJlk;52q=GG3-vbw<o&=()})csjEg$o}-A&vK(Hp5uU%Gc|D?Y4JS84T^1w
zn-m;&A|z>GpzbgEI-I~0apujFFYsQW<{X-Eb@E}s*C6ODl29(TTaSmD)v191GlI-K
z$V?9<8CK58HpxNcJgkfm4S+liEAuQIIfjb>q@>L9h;jl@)srJi8TOq8d$pY;%v(xe
z?MaO7M3`5?$tCJe3+~Eg@)SxD+mx%}GHZT-;zJa-C|;u2p?KGXk_vr~6Uv7uW)f#j
zqO3B;s?X0PPfl?L89!-ImDcbLKHowDLDh8xRjF&N$f;;n<oWCscN`Ae!u5oPG7w!J
z+Flt9B&lt#%(7SZ$AcCF*EpxQ=jnv&ew{7TstZAGL0l5jMOZJ%6IFQtnjDjTnq}mJ
s63!Cioll`mabd>!XOhF%XYnpdKwHGi;8z0_1BS2Fg4#2+FV$ZBACbe$DF6Tf

literal 0
HcmV?d00001

diff --git a/scripts/model/__pycache__/Helper_models.cpython-37.pyc b/scripts/model/__pycache__/Helper_models.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ddd209d845db72a9645ad2bc9eaa9d5e1dd24ba4
GIT binary patch
literal 15447
zcmc&*TW}lKdEOiDAViTAMai;*Shh?gHfhP%$VzNWvh27H<Je4`B5@H2yC6Y=0Cjeu
z3j?-_C0A-vr%5Mi(<V*HOkVoZX_NHcrfJfdX8O?SOds0Wr_A&rqnGxr?PNM_zwbZ0
z3y^~3TIo!evp>$BbN1}H{I~BvhbPCz@&>N&r2q5m+@fLpGauqBjl?s!{S&5PxQ1)C
zjjH}N*Q}~#>a<;TP-eBwwNy228qXW9?K)Qt*YT{C%oV$ubyIHI&A3@Nx8%5aJf)Uw
z{1)88lHGLNqC0lgtmfQ`JMQj9DvuT>+)1=B8MRPw_f(7SRCNp```l@iOh+Xp<m`86
zkTVnI>_N@}_aJf(MmgiixywC-oI_F0UgX^E-h-TbqMQljyvr>kryS)>BIn)iy~w#Y
z%9%pWVfQ}d+!y8SL(cu~5#$_+a;A~<fIEwv*(hhfYrbk!-gAR<x==C0e6!bS1g&nT
z-fl22zU(OB--m=}R87~YTGx!G>6)%}HCwgytX7?ChGSGyXfqY&U#p*guG8qco@!u)
z<l=7zH{0`n2oe}qj4gB9Fpa=kvDq41;+njb>KipDu(ndJwQVAuMvaVKskbaVXXOnG
zYh`1#if=i{%Srvb)X(p#Uy%BFcWkJBQR<IL{jpv3OHzN#ExCJ=_V-9$3A|vtd!uiT
zy9UNPv2cTfyD?6(ha?5kdInrtvBI<;sFv%6nf1EswmM5okI3~8PcLH9Gp){gyWa3_
zl=&Ljx?-v<)6bAxCYd7Pgxz34Wp9}Bd+VMGv$a~Q(+X;}(CT!;^r>#={82Zw>+Pdq
z;bc8%EH8A`n$9^E=3esZtCwE!Uiv6&JVNp)$zvpslRQE4B+2_i2aP-|^e8&%_+Gn-
z-(0QMXxDweR{I^}CvVKRyN!DLA-|#ct@-tp?wWV0zA>-5e#@&Zd4AASUhS;cX)LeR
z)#`ks-R*ep{Hb=&4?Km5m|sI`{#5s&SJyqSv0QsTdgXbq<DpG(;yfB}tq1=68aSyv
z|AN=XxNGv@&#hlllbDYjZhr>EF!T1bSv1GZsU37<a=?sp@su?zzJY5Dw}bl>ZvQGs
z-`s*e*)})9cUxwkDRA7D)dweT+DO@b8!2$ymeY5T0<UhR`YEK+NTvH}q%ufl`Wd9M
zNM-w3q;g2*`Z=WXNag$B)Xf4?g?<64V!yamqzr6Z;3#`Ru~RqL%gR`o>vjBdJ<r?l
z)HHGwhpP^f+(mMT<ZhCCNZv(KCV4l>y&z#OQ1y<#-u1o8m^#dy?1lRInp+R*Rj1nv
zjtV^{^bw(t3VlrI<3gVh`lQhJRa1QHVf}Md-On!xIl{&>&91soS1$YLlVEYIygA~}
z+L#zxp6Inon#|p)uX$kZe2pvKYcpM})z0<mZCxVEuine=jYXW<@Y?NKZO|JC(|ssc
z?3AM7p#=Gy+N@dJD>*lqt;hzdKtj<~dr4SLJpj^RiWhl}w=l~$kO;soSHLK)x!+i|
z{Fwj?c*T6fbnQ(CWk`eCS4;<5Z`D#y_F3y>2L>uI`wnX^8Jnq#F9%?;E9PnA_0wP?
zvkxY6o&Cm})&b*_)@BAwlnQLHTc)4wTmAF_v`M<{AUCybfTqb}OZF3J?cB?+A!jAi
zPko-sDOI1u2=irJ&=Al?JyG)IZnNBg#+WO&>z6#`&y^RtfmaTey>g?cl-CK$t+n;G
zw}zC+V!7VFP`~7ty=K!x3+KJ|r7|SEQx3Z2qYs^y#vYW<w7_UB<+<gvm&(gQu<jq9
zpRcQnt@CqTwKTu(Hs_B%@$jQ_4?lYB*s(c0&dFQH*XydjR_@fKKgY|R-r8AD?e6Bz
zs*UA(r{lGcms_2Au;Y3!7=BF``eiUx*{e5}^+?OJ<+j(E?bM=|E9LjcA1%MPe6(EA
z-Qj%kTEv7jTgng0QT=kumsuM0uc^Fq9&G2O^4WUF_1R@;b@atw$4sg262|2Zcf8qZ
zLj%_Q)`oXn+R_<iW|U_yK2*Qh@++geJ@o2u$1B6lYkPAX-R5V?`tg6$xUM;(UFC75
zkC*G6a`)^C234Nzbmm_5pp!a5tKP0~^)+i@*)e{Oywz;sVVK9HlZzid{o&)1;`Dyv
z;_GD>%+d(JPCef%qg}8m44I67CkqbVdL7)@?q0yF;6`EAI(bqU)kinMdo)Vr4lH23
z-P+I`96LCAF+{#M<`<W-ptMBgX0MGY12^4{KQ_(?M|Aml*su|A?p&hxOKZu@CcGw!
zrIHP8d@L+I>-(Of#d}Vvt_o97)=QqE;aB@WLWhQ1?PoGSH0`-C6Uj@MOBgyV4D~>K
ziE5_q2embSsaoK2XbJN}vsKNvlGmyQNKh;k0}U#wGc0gHYkc{rONHyx3>kZ7OvgN8
zJ7x(c*D|Ngv^im`<Lspq)H}<IiQ=T+N7XBe^D9W)N^$n#1Fok1St!z2JvtXZm8i$X
zO|1ycehNwul!_OM&|0GPwbgn*y~P?-qH!54xCvFP-iXRF+lJ&g{WR32bMA3bm)X7(
zsmss8mlI9{A8MY)x{#@xjYL^W`K>k9pytksx7Al2ucz?E(Q=<G)39`kx*Gn%KnNhK
zA|PH18eNqi;Bq`rh9X>m6DFVNbT16QD)K^y@pySr^}PO>dfWFNgd%fWjXDfNeSIAg
zI!db0Xr~)UTf1cY|4&LQ)`EHpY_@UthdbzYqDJD0ph>_4RC4MlYN=x+kC0I0!t_h6
zj#pP<8iV#au9iDz5mTrhLwO~m9%tPrST{eI^Dvi8Vl~%nL}I4i$7(6DYP-eCYBpX(
zsg|a~j5%&mmJULyC|M=5XsMGZZ{SWY7<0M&2k~<|I7T3K6@W}&SsI=|CM*nGtz!u;
zqihm*uwnm8AOl=Re{_j>3-^lkNv7Zd*Bp)1mh4T_O>J7h(1M&dZ9&r+<U4M9Gv#I}
zl|c$Fi?L!a!I$1o$Hloqv_@YUk|=G$M+SP!_u)yeWSek*zHGTU;&8!r1;JRw1yw@d
zcKc`#GcR$}6cLy#{9?sYPxIj!kV;WaFip4+Lgm3Wi}9`IJ2e{blQ0<7B1Ej)3A~HJ
zYpsq}Vi{l;LfCK|Knr%J)@;}Hn_(_VRwtrbHSK}C3L`G77UtuEW&<p97$2{W$w$tg
z(ymFhpdVkN2d1867pWwx(B;jBWU@xH5mk?!*~HD30l9c1#y@bdOSm0zv>m8pu#~wm
zHMHsj07S&fix7j$e+sw%3<wqm-sy_v%Cddk`aIO-rWrUvifWaM2DT92beq2Kj-2KY
zctLl$FqN!&4nJWk=&HtY=s^GY>NIogRwn>-T@Rgg)pda~F>_cpPE=Tm=T1-RF2PY}
zP)FewSKff-**LUgkh_*EL3`{_GFSp54mQDfX5ZSfT)6Wq_O?cQHpIqSxbfe(Dvpdp
zy@0nu=X}e%pq@t(+0tS2VrVW`t>t68yGnC;9=Ff$0tT5=)-WPHHWy=_jG{!Vhe+l~
z$gX;%Y(veH$l8;{%y5LcT)PHt`38_pDeFky8D=II2hYo&1&K9>o0U)w!Y)^H2<7A`
z!U%N@f;stc-*Z|FURhYEy#{+yd$#Acy0W-GmIjR<J-yJN8sWuX-ZpN38eA3->ljxs
zwG8#ZFjI+$8lhsSrTSVOFyxZ~?lD$#O7+t#S=XeA2VOc8<lcZ%Ko0Ow-$KsEnG@t!
z3M<86Y^y}b&CrpAhW2zyK^~wQi@DAA_n_W5z68aCh0P@4rM<y~`a9H|gv*hcLc3CX
zD%f`gOi`Pbx7g3!Z|ztA7R(IQLCadL?QGrU2oD5fD+jmkg2Y&>&KFEI;c}D*M4@FD
z6h%;`@eG11X=ow<d?>Ck<ErjD7(X04+tSXlMCG*0cu4EAMZPN>BC=FiZn>`45w!-z
z1L)-HbT)Djef1jNK<H?>ju?s<Jh64T=UA6<sc=tIb=PX^aQ0Ol2GCU><Qs)E?QT6-
z^g2Eq)10&{we9C$K`3cS_29jdA8Kd3IwL;5I)T^dX|A0`E44PXDfu%Tt4p;=utVo;
zH&_l!1F@|kOcw5QyP{<Sj70#U2MFVbPNV16!=me*Z#6vV>0T#@RH~>FRmBI?&yswC
zM25N#BSD)olv>j7SZ+j+M0hxfmvQ^FB?c5|I{ve=<^=u@S|w`+UhagM6$R_yw`7*=
zG?nW;BZXt2W>ahw7h?+)3r>X({0}BuG1NGysRbjj5k7F#xBB)<3bvE}Y+9WN(ugD3
zP{k?NK43J#q?>l<;83mukwBCbWzbm?pUO&nDkF6e1aRpo!xcCHCX_lEeC2nc_j025
zvO#)91ZiuG!3DIDLr$rm>t~vb$7bF_P6wd}iC>w>ceddPY~~RG%Oi50ebb65#<J&4
zd;{OxQNn!IW66F4(XV{p5krxye-0BNo+Ghf<WFn+OhY_?i7ftDUA5|G+n(S{axp+?
zZ>@%4;`;1j<w5bigy&~ZRw@sc!L9-Dr|UJ8SNFl+bYCwKjV%Ss{wzJf^?G9!1sXix
z)Ptzz&Q7#j0Ez^-f_)oa)ZXqfXldN}Dr2?4%E06Fr_lxkXz-qf*#6E}P@}93pbt;H
zEV5gkU3~E5gG1sz`$A<f0*(S9+Mui}$jA6C=<eCpoJRn|0q9r>=K!A=ynRN!4E7W~
zbK}2Y^R#D0H3bD_yREf_%D7r$m9rq>1f8IrlB#HNlt=PO5>X);Xa!L_F0(#GBDyEl
z3tDYIv=zeFQ(O#)JhT`zS2Vk!4eTLuT`6b>D)QdQ9pRYflxUr*1^p;mCQR$`R|{I{
z$mgqj^@FNksA1?e0Dx*<XEmX_!mL&jo*Smw8T@9V$sd?q9Z94;-p1`yff&WKna^ac
zG@N(GoVMY%Lru_$&qHNQ*^V`CJ!x*-Jz7!3*5B51{ZX`s&6tHsdZah-c8&n8!Y|;C
zLY$>{E9P)blFN=#{(VTqUcl|Qb5s*m6TConV0@Gf`QHQ?;05fc*w`g;h#P2-wXu!&
zaZnQNw!LLZt)cy!c<<(>IzxLl@qW!sb;bt!K--9|*D|P?<=%~$^qiR6aUD_D;oUTP
zt*DKoJsNSl6(hUDn=^|0j~Z|kk%Hx*cP^pg){ig?f;KAo0k+Tq8$v{h+6Zn`2k;n{
z$Y3=vtpZ%^Dlw5^4&0?dBAPy|zraP&+?Lp;e?nFqLkCmQ3UQNTK&P#Z{X5O*!2Hm=
z=f%aMDLRGQC(;<T&4FpN;VE$eVNhIK%nf`fY;VvVMhV<uCz?pIfv5|xDjiC7k%aQD
zIwT_gmyoL1GC1`fq(d8H9wkjbMxihJy^xx`H4VBky=zdhsN9)okca<~UvWS%d3!Wh
z;~}|N!b`+D77aHo!U2932bqqtG2|WtR7Zkcdv@{TH(b2lAS3bOFS$%4rH{lwQm)g<
zv?Qo+Ue~Utj%;P4r!>kk*NkiCHS3yv&1u3TaIxDOz`Apz;0mHzXd#DM`Rh7e2#Q-{
zxJ%a&4co@w)}HHtZm2ab5@0VRB22CzFVpq$4}mP9JFiax<r&wl>-Kf$x_RBWlKP@`
z&Usa{W5ui%qj3r_g_mm0Jsg@fZ1$$LTc*9P3>0q-mV%MVWv}k~P^L|047mCy`4cpw
za1hr&kOdq0>`pqhF*`Jo2?x)ekZnf)14zU#RrU_a9Mzo`yw{Nm$Aa$az}pMQu;n%M
znE#OMB)x=|B;FIIB`)KKnbX?kmO*H%<@o1RplWQ?3cB!|vD@h(Vnj&|bK2N&T$k~z
z-OkT2^#%zYP{p2w`L+7Rnl@39>sd)jiLP^*6$kGnLjNDg_ZiJFa(P%5xSQ5=YTPPX
z8&jiZsX-RwMRuch6FCo%=$k9R>jw<DhD<@Ou`>mi6EGK!+!nUo!6*)0Ogu3F11u$5
zn%u(7(nUmG4)};$me}OnFVTRDGsrDqzp6wxm00VFwP~LFx!3SL=m>0Iih7_xA^0-z
z1k%{!VPu;MBTOrA9cod177s&Ha+QytBYBhL8c5{phqkX8VHTveR`*xK{J`f6Gt9+2
zhc;GRJ<B)mPl!w_ZRaGq<ui_CI8-zDI%TJ5mYj{r;Yp44jHxKIcW8+}jaUC2w^lc~
z(xhTe19x)8YxWK-aX3CKgw!|<t~M|JQp2-VpKjbh*@#}3m6Ssj-~>?%sX$f^2sb4m
zxeSzxDFHfSS01N;qb@ad2o7G!P#{gaa5>Qij2nkjJS9+rLl_}W_*cjsYc{ow(~tW<
z)N2Q=7oZrPrRT^k4{0GR#c%7(@X*URM!@0=E#Euid2X~Sy7mkfIVr6*2gT|-MlPa-
zbvw-TWs-q}C6YNB2)>sLB663`TSYOn4z@P3=Ei~DGoPraMEE%UiSWIM2M7%WLjJ66
z2dD{=QRFd)Ckk+mK#LQeWKjgaIX9J3q%ip@^_v{auan3yzQ)v7Ku{!O{Q@(-2vT)w
zkO%eid>$bNPjQrDZBtYBR4Q*zIU6%KjW3piJF~Z6#<$<Y?@$TA-i|BgG;k*uS(Vog
zd%MFG#WpzX?Ql3M6W(<T77}UT67b?LZ+>D|P#e>0O+zOlj&v}v5Lu#$q~C+UHp>pu
zG($E3yiq1DBtY`e69DqiQ+~%&0Z&CPEZP_gN?Uty6Xd5Nle{;Qs0lI2*rmV>8I{Q)
zdnTco%g7a`4YM?&Vm}F)wBzVtbKzV_x+?U%(C&cuwHx+Je1Y~$Ygnq(m{Be>_cJ7K
zBM+}~1*cJw2ROzphDP$VMzBXETBG}JvlxkhEHvU}p&w1J=oD^$3T-Fm38p7HZNr@b
zecFa5NT0UJV($)WN<z}wNI6>>ctnCRwjCIu%x%kYaN@S*@T(9HVJmEvxW(RvdBnoy
z?=leiz&S#MCBNIxA+QK^vx+?riIikliv1k04x=P_X~RGqg=f&RD96d3;a+Z)whhUP
zN3v(zmJ>kZasmjo`+13r#GyfIaRK8R8$OK3CEVrOmi2w8uQt+fg~Yo4Hmh!tY?FM1
zMArA~Na5(2r`CFiq<xT2>9u;W)@zH64GXlSfZW{qh7s*H12`<g6=9zHA+4oex94ju
z`VG7d6L6S~q_Fp|E`|20cPUJrueW<%=&&jEt1LA?5}H2~njZ_(Jj=CoDa=Wrfcjj$
zf(oOQQj@G!Mj==&EhmN2WfOntdz>tG#!AkFGv$msd1vGB?U*jsT6c`0e+xZ$g?)(>
z*Q8=j19x&!tMGa!mb-jAgC@F*-{Zw!-u#OHILK~G9>uD8K$3b(>!=i-umvufCnMSg
zGmsrpEIakbQPCO420A0!$j~dDII*Lx<kvVo1Q65;)CC)rw;Nre43fPVdi``)PvMq{
zgtTL;0J0xhLt+>?g$K1Psl*P7(h^>Fl-KU5Z=nhGZIEdD=$1V8O?F&7CVQHkH2b#Q
zj@jwXjNtF1PhTD4rL1Dk4ql2^eTNZzd(e5>ldVZIj<~6#K_^}70r&i$g3g%l3V@)+
z2!bXSFM*(YBM7SP-A@fcqfv_!h_&9t@Wjcv9j3aUm5z{H0r>$}gRDouIW&n0e~tv4
z!v>nLZU8vdRX)B+@{1&okPx$IQ=Nv9X2>IPOK)wOBV?J^F-I}^m)VZ&XncmLL@G5}
zdJB2)1XF&SL!ZD)J29o$@u^W$k(liQ-box=^3;&oj!|VgLX|L#3997r?E$LPhhlF=
zl{h9Fp~`ZEDyMPckyu1_?XJ*q#_?G}mj=8mDg$mJ<l^0Cj7#sM;cRPyHPz>tOQ5NK
zg@oO{J<3#nh$>N_IkW?{6!HK79qBNOC&8TJU27YcVIB~t`Vw1ymgM(9!n9t%Q6}#1
zxdP*;WYEAx@N+r^MKn1)|FO>bsQ~C7;49za_o!g9xzawxoE@siMh35UV#daYMex=D
zl)lM8@!S%C(tv3&nhxl0TL~D=?t13=kO4Twd44d02B%}Ou`BI1cE5&B2wVy$xsbb{
zsURoy8>~ZI74AhHp7vlp4MNpdStuwgk>5LmmH*0xpaI6h3#?=WcifDj<?cC6lu;sq
zG`5KZp2q`?CGiO-MWAI0PVfL)CXpI#Ofj*T+Nd7oFb0{!^->~Pbi^4Kpc6hQHMt$A
zVQ6MI4E;-VM}3!M07$bMkp3}q|AgdENg`+p!O_r^@-j+JOekm_rpXYWMi@1<F>}*I
z#iDX&=J_x1{eMYh#4JuI=8Tq+M^O^Tgm++6U=T)xx!1wtR{Sp&{32@X7}X4>EEg7V
zWb+vAo5Y27h(ai5xd!6G*~gJ9BGUMz1kXs|pA0~8LLy!V>T-Ps;5nplY(V=2IIW3T
z7`D=CE%#zrpqh#vME}Jc%y-mZk$j)z2PCv@qg}rHxFm#c<luORd}C^8WCNz<uy}E3
zyiehl9Sja^#aOks@V`R{J0hYDRj#)cqW^f%XB;fuB`WWD_uvaiYvp@CQ$lFe!?Csc
zsz)U)`uP~UDJ~Tpyp03H-P=(w@j@w7iD12mhg<ZK2cjg5%9uTl*QXFD!v+b)lXmU5
zFb*D!LUadW_6!c;_xXQN@m9$LXcn_7lbh8OEp{ofRX8rIT5ZM8OD!Ib=(gj9q7YS!
z@iGd?`m-d<@150#Nd#=<-0)4l^%UnOk2eiEzZPKCs2PfDH@I@VcBq~c$bf3VEMesk
zCIn{Z8ZxQ{hY+gBlHf662@m#Y)s)8n!vI^y3dB)=jaJm(kUU31CInPD%(Sets0}I$
ztu{n~{5c<Qr7#wfp)szTDmqiCjmhD~kM%@$<hJUdKMWGz?xfZCP=1&n<E$X6Rm$eF
zlZv?w+{qQs-8-?Y_UVO%B$~_S369uM6u%liMTP$w5glE#FRlmB<?_EHE~Xjb3;dtS
z3v$$1j*bv3B90GD&O^`V?#2I*K<sAt-zc4=7Hz}MT0BP0_Zf)UQGy6R>`YB>s_uyM
zM;oC(M31#n6312i;rBC-=1-hr?TVemW4!3B85W~64LzKb@YOG~_8*amzs=fml6JIq
z^0*{oUlE$(!N(Fpwg(ZuJ!l;=H;S0q(b_46W>4XkfceE2aVj0>abz-2P%rVxHD$6M
z6(7b}frR6H3+jOD&2#cUgs8PEH`)e4c#o<GaxAkTqcqPT%F<d|>$Y4uC91x|8qpT%
zD@e;IM;S%MF^c%Yb10e9){MDP;K*X$zB3!h^>W!Xy9$mTQ_KOldI~>_l^Go~%F5}@
zxlTui{Zlx3ga5yK302rFL;WpCnAW>%@^4jgf>w`^6FhV-WRZ_el24N`Dl8EefyUyb
ziWOGhV}WqfK&wgW2nxdN3H?uDAAtGR0c7-@o8r#I;Un+LADB4kfDtA>K2e@{WMX#W
PgA@BFPE0&D@$CNqc)IuU

literal 0
HcmV?d00001

diff --git a/scripts/model/__pycache__/seq2seq_net.cpython-37.pyc b/scripts/model/__pycache__/seq2seq_net.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3291df916a23a8eb5c7ba618cb0916e533f5570f
GIT binary patch
literal 7713
zcmbtZON=B}R*hdqMn-<By1LqKw>_Sm5rR(B?t%d^qoy^syWM32p3=~6&!DueP;^FA
zRc2*IWWRXXpHh^BXpd&3p(U0BtE?s4Ss<|k7HpANK7oY9V(1+U7Kja;^Iqhq%H;ux
zD8;Ra_mBJTJ@?#~ciQdH!u3;UBYy8K%lcQoEG`cZ@8T|g29a2aJ+}J%?~Lt<(|2rB
z=1#o6S3TDzo@3e8#P8RulAs@aU?pziePJbD>Wv!D-F}FYTH>R`uS%LIsV4zSf~us2
zl137uB&<r>C}}1wl(ednPGWy-_1Z77I%}_ItKbK*7{(9B>3}7Ri(kR3xDJt8eLJ!G
zP7?gs+P4!gseRGtyO_yO>R;G>@5h#B^=lY4RN=jBkSD3Qzqj|&M%Q0(5X<_PclHK#
z^JwuuZx?s*R}j*Yc4<9#=FTb9P~tgqOBd2B9mrbgLHcD4vR?X$b<_}|^hZHrzlPRF
zPO&Q+e`4zr2PF<l{)i<q95qKR*`9Yw|J0^BUvRvQtRusD6Z1Hy`W@{yF_X)AMs+Ir
z1?Reya1C3sJ-R&KKE(zedE(~@Yqk;#A6wrO{6axh?Lv|MzDU#K^g+*2^+e>;d?r=<
z;V>DBbRdU$7LR*ZRIQjzQz7{6Du|+CHk47M!eKU@$*33}r>c24Op-J+FJU@)m?q{q
z$f9xlEER>I_At_6JWW;Sy`o43$G;~;E|imHs&S9Kl3{kBHugn6iKZfzA|7UGqG~c1
zgG1Ha9p|xpkY)w?H1sUGz5BuaPn3HgX6j0kNBiSgN~~oXi+GYs%&ok^EQwVsNgodf
zX*9@Zne5e-SES>8(WE-s6fKGlg<k6wEFh{@5G^Nz&KglP7{^5sMSp4i*B|eU^Fcg*
zyBLUA4i0ywqkNJ+i;s6iUJTRdAT8t!<2+2W!QmtpM>~UYj!)XTJDwF-8$Neuf~TFk
z`P&~)({ykceNc7zAk9*Y2_@}d@ZnSzI}=#xc&A7oeG7M#rSj(VnV?s1;4a>Tu<VZQ
zFaDgs-ooECr{i38eAFCYURnCh#YPSmZWExpkNYm};?E#pS(ey@v@0vO=5A@9+9#g$
zMzxVYs?P)1A1v|&7B;tG?e3}bnRDWwxXTs|+OaO9AJ~cgBWEEoa`+=>--Z33sbM+W
zvF26l#4r8Q-FINUPW)3?Z`iG_TY8JTl^v#!id)Mnwpd_9H{b7SyX#)hW|IhNpI(2^
zyYZ0wrLFY(onG%o7e*!1q?@Dzk;Vl!X8f%C@L6}9W(V@HxW1o@?lc}8p`y2(=FAGJ
zmS-0+9u`tBAbm804W!k0=SML9bG9q}Y#5Kbg^Xp&F<|Dae(;W&$j*H-T6E#U@XG>f
zS$y?_8+UH3Iq3Bdd#f*?qF~IjYZ~auI2W++`tW9Y(^Q%<yVL=uSoPlPHN|VN6mb<o
zZGGAG#4D`brntl=wV51_3*`!U{5I7IHwAJ4Eh_z}%P2tM;ApK-yJjyCZ(xZ20(Y?A
zs%`eZ(!BD`NAv^Zu0RgojHK$7R{9<D5LI2o*SRz;x?Eo!pZFve$*<t0*o3fJz8yLt
z+-TF@w!dQ^Z>_md)wt(sJQX!GM|9fj`Gm;~=tG!{BDn2r%;#1VJ(|U1Q==VMAQD@u
zT_MUK9gm}EwWgje=+vT1LHH#)74+A@?p)C&1H3FRHsMv=fLH`T9scaZ`6590)8V;~
zV921{ujwG@GcG9aJt?z+qx+jUM=aoo#XiJE@lJYk`2EOD>}|v}9hZER)nzbm5GPI<
z6~n}l&3TJHJ#Qy2OJBE65W_~{s59C?ymA=hNFuO~+TKep;Q%+Te}6|uD+DKRlE#^G
z3z_sbMFUl0hl1E=a9dlK{}IAPER=@`fQa(_MFh|Ws5}v8iLxJKuvi`f{vQbm$@t|Z
zv5Zwc%f%!fN7zqVnjRPpWb$~i5lchSGv7v|f>Fbu?lz3%75lhNMXu0wu_ufvG?2Tv
ziyuIg&KyJwXbKJB$sw+mHc-?}9K5-6;-tY=u#D;%dIJX6H0DCrqZ*LYN!%CC9RE*A
zlDxQe0Y+YgFNklChxx2ncYTdB@-QU~k`momgAo(R-&%5m0<iMo3JWSM1jw;tR5{#P
zNUJKB7{x%}tsH87Dk9eU&<5<5;78EXL7vgE%Q$&3OxF?b&W#7Fg{>G5g14ItC%wA3
zg$+=?NT+EmRRd8zE2epof}jj?kt%yv)r)*zPU5GkG0O^IH&_z-X}kC}cHTJmp{kC?
zgq8k=JRBC$D!eHB!+x6<87)@2@^ItY1cCQ)8)IsOb_hdi;qSPmjj6(!!NQhkM~v3m
zmil;*jK0>~!GA>P(m8;=oWdTob-8n|vL0~5QEl#*&ZrJ1is(xFnm1@Ui0RfmL^MYP
zhef*c=E9!##BX2*gK*@Iy;kK;GR_XgZ=*!HgIJ1p+3MRA?@;_M1$l|!mnrw*WCER5
zima^^Dej@V?`zdOGyTtEvUm>-9^fwM?Uwxp^uqf%)Vk3dhMoC!)CuCAc$-2a1NExA
zi<^8}j=ON@E=vg{Tx>zS3+z@9Z0fQBUY8d!msdeuERZ#6YHTHzGPTfY4Y0MP(;0gW
zUEAuFJMm!>XYq_{e&6^OLaf&6FaF}sdHwY}yL$ufDlbAD5raEk{3paY{ACWX$Tyg2
z)O4mnro?=s4jEvt>*Bs_U{`@K8l9VXYn3&P%Wc#L;!xMuM@X<tUZH!5(mAndM_^wZ
zClEg=k-2;uZ)m@X`%)QjOgwGjzbzZ1%NmF5Q|2Gr@=EDEC;GpFnpcshIIyf5aGq<u
zs{12%(KEiLvy*G`_0b#iuOVack+J+o!tc9QWh^hLln3zLI->nJ9;Dbl>SkAKulJ^j
z2Opti6%SOiit~hbP3l#n%5S2ndNmn>QX@BrfB+&b(hL*}P@>ue0O1JerJ1&-EMkiT
zlg6m}`zAv&`9VF4p5!7a)TKqIWs;!6k2RekeG|XMuFX4Sn|p{aznLiSe)`cTDr9~!
z)&a`|GrE#^l|r*A7hhn=m8MVESn)bUuVI`=CoYUW;`<ajarr|&-KY2%qJPN@kBn`R
zsVyx3Y#{p_jf>xbu$+#&Odo6?&mFsk)L|1(o*f{u2wV^76xf^KMB8r1q!nMk*t%Ci
zkhsB%v7IKZfs54U=~b8)!0QzF_j!1N^q{mO=TYm2h7LG;y9mY_W5Tt3t}UQmq^L!7
zAmXIwXoaYnZn5%mjjgI_GTev&{1eQm>5~OEcl^z-SoK<m0g;v$^}NiD0(=gaeKt25
zx#Bi37%}|Rp4a#xCMJXPxH><o)ALU3Y#TM_>xtg*C}OfkV0bb(lsgA|GDv;`uU@>J
z)HS2B%lbaJ6Y!co+7^FPg4Z22in;9IWWkzmlpa>HsnNZmdjw@;)q_4f-(oFR72w&l
z#8`+W+Pjgpg(uB2G_6iT)Lbq@v<}O#Y?l7M3%=pUHg<IxnkyDC$>~qV@Xk^IQ)7}g
zolp#BLcjsLayUtmex%biBJin>qTQ!mMD5Sd&%v}Mx`^1E&u4xKsrl598qBbm$qcCT
z!Y|EuF65v6u=CtznSXuCk1GC;Q#>9f&{~3XinY}4)&=~hB4`zrTIyymm+7tUle8;l
zS(ikli{l2ZRxqY+Y_zNsyJz_f31U7QC*4_*q6&DEitaeir_0euv}E3$iX7`HFkaey
za+r@XV3n>dTdw%Q%m2ouIm2FWdzBrEZ@`5NHfb>R9uLzenulpP)lM#cpKacy(3tmo
zd?J#GO$xGR%_amnkN5+MPa(ia$jek~-Re|x%|0~AP|Y>v_S*#S>RcyH`Yqx^^dub~
z9Lhq~2YDvbr?THbqH3(Tzp;S%s!sVP#0b*CbEu=!x<5s^K8mz9m_z#ZHDKwsyNNWR
z<!<7=1r*wJHto>e^uhU@<2SzwkxWwB^Ng0ihk<u>qIogxiMSxmeSQh;VZ8i|D`a#9
zqS>9d0#C4XfO~meOne%e_%N9y@iEDymxP)EtrMT7YE;Kqxd6Lapeg^8;kdYqU5E?W
zjXfe;03(I0NwC(^9)a16$YO!AqkyNMiT#3Xr=humvj-yhzsJR1Xeh%0W;T>z6+w0I
zq!vu}*IJSWkw3%{sbOwC!(WkSii4sbR%Qo`TE*Ur1MT1OgK7RIV`M<}mVXWo7na)k
za_nRbW&FI0yI{0u2x8={QfHhyIAVx$!TrblI#*e8UL6f})jx0mj%3-#S5_0QhdqWm
zUZh#eu*aOK#F_e0AUYt3e)a}eAg)Rq$JP$E1slY~BXM4!7Qw>f!5xX;sfR$}8n%mt
zeHQ~kg<93xu~*K{^;c<@vX8|>yb9W5zjGFQMZ&^uLu{T5<zbZN!y=6y<Mi&_C({xB
z5qFh7K<7NpagHyoeR}0z&i~u1Xi|U6$*46P(zF9tu<T-=`c>2{lL=q@6exi9DgItz
zt^G<-*PQZO;FWu+T;&Pcc`=Iiq&+-H>v?4qQgd$j2{J<MFFY%-Uob=PWu5H4tYfR_
zUGyaQ5ZWQeM<HsgsHr{5rBTre#MdCOcSK|0Exp&c4rew;Fq(lL;#JkF&C;K)ZBpTf
zfPJ3F7pjS)c`#q5+l31FFJew^PWQx!!%ZnNiccW=8!M>Rae2LfKL8YFRn=jc-u^B-
zhY<ga+Z^P9wmN{^4j>ma7H8qmWu?!Rvn>#cKL}8_ciH{#A;8&iS_8Jk%*b6nyAo&P
z8k3l1gG=EVC*wH1sh93309WG4I5Z1xXPvs$Ie0yx^VWF!{TeeYoS@-<qii5YvTs{Y
zKa$A4IqGu<wD`7WUN!V-fS37Tb+ymjljccq(gF*smEmm*e9RiPjx6zW>}1HX9vzoW
z${$)~ZLwP8gv1e(Ikm;Re5UuQ8q%b^FiNMR8zdnvDo5tZH@}^T39Go#s)jR@1N6aj
zuWjNl^0P=sO&rlK4p{p&z0ulVMUTDjL9E=lI<f6k$XfBaetYq*JzUwczieTmuCmw8
zNe#Bq(f<v%`60x?`HAyQ7w5KaU<VFjF8%_ze&__w@z&brFHUiJaQoHA2sgR*CA2Ky
zz01-u#6RI)oXj?LNpC}Iw%^e|UfqmmGH13Th`cH=g;}O^%>L!Y>&^XHR{ij%!qv05
z!+E~Lp8ogrX|g{2HF=|sRs<P6?P`tYsmo5mZF2_wU2cA{1%ZPO{B7AmW9#j$Z)~Mo
Gum2C?k3y;d

literal 0
HcmV?d00001

diff --git a/scripts/model/__pycache__/seq2seq_with_cluster_model.cpython-37.pyc b/scripts/model/__pycache__/seq2seq_with_cluster_model.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e6fb6db5f42977c915e169e258efbb69fc83d30d
GIT binary patch
literal 2283
zcmZ`)OOM+&5GE<vlI2H|&8sbv)aVN<V6#SZ=phLD*fiZ9Qlr>xQov}TYDL=Ct}J;;
zy6JA^OOqmpBIsZ69{V49>d)Y*KnoQ43q2H_k>q$2q%t@hk~1HN<b30;cH1NH{qWbn
z-)5lwiG$^5gYq#n^(6>SI8Dd^dzR1%8!%eVSrdC;S9)z?GeXIvK5(iTci?UlZgKlD
z;kK~%8;`7k$LT%d*Pa0?>HE~U4?-D-dx?l}X8Gan_^J0m2r{6Y3>bHxkRjzZuRU%I
zEMDi%V=}Ox5St8YfY~q?qhzMEkT*s`>6sM4B;z7^Mgik*L?h&nkNQ!gR0;MDGz*&g
z4TL6|=HwAuuoE~(&T<l1c@3nU+aPPX1+t#kK{~ku(#`391A0$47cH>2PSB?f*46=2
zySnou)y@4gFt*Wa-{q9Er-*nKJkCK|1AP00fp#8xw7vi~)>C?ddtLk=?@NWJo!5%}
zE^`8>xBI(3GY*&8G@EJD+zWLy4u-K*8gAK%(`gb$!hg-w)NCrG#8VqL2;wx>L0~*E
znCU>p$HFuLU^N)WoQt$v@x)|L@KSfvAPJ9zRI-jxP8v)zC6s~rp$sRWu{1S}JLtjV
zbYvQZbD?%VQA$W1XXza&Gij{to%_a_!i$aM%y{?2;Y_4D4ii(q8>b?arn!aZ>1T5C
zJ~#G`xO?AtXrGkJ7Wy?H?lLCG1zU+^XzWmqlx#wPvV)?HqJ`oNiX!rhh@)i)NIZBR
zBV1Y|2%;oZDhPfh|NPQRvM5Zht4Lxbz3F~75l7*1Pi88HxT}oY6KOP_gz}&lB^dzr
zZkH(u7totPa`tX#*Y8b*fVbJMT((6j023~w2jKBkt6q_}JtYorKnotmdJK^isa>Y@
z=JZG+buontiV)hN^;18Wb!ZEkXW7)FE<KAnJ-+rzf^06cAi|`;53dQSaT}WY9fT$e
znzIwSU^&~jPUsxcb7AKcG)V46jh7l()N`A&6ZSQm*XH%oi!;uCVo(N(erUZ;z{|<&
zbNBn9b~sRO-+jbBCEq-l1DO_$+$q+b3KfX9@PKY^f$GhhxwmK*`+2a^$~~}d<*mHJ
z&l}5qZjrn-cMkbiB)5<t;m=SAyl(Z~oQ9C4-74+Hp}P5e2`!dYshbVEY8*~QcRig=
z0;n$2^<96XWEZk!earVZx+;@e@GcjT6rqC0PL8^JN8LoEBRy8@!%TLkVRQfs{_|~K
zh@i53F^D8qx(J})LnJD^mk~;azUYdy2L5!F4z)lGVA#q}3Mh~1OGgHaZpnNF*RnkI
z?#9-}sS>dMxxYFAUK9{ScT0m}Gu&&Z-(Qb6#b&u!Vs`NkV2hQv#3J99Se^a0sp(8c
zW8)nQAQaI0Kp3VoQ|IElI1=&#SeeEwRZtlO*j&bOO@`@6$aA2}D=4nwth2Pa1|z>M
z--K?^E+4we|FTq+^~$NvY}}Ijf}0MI82&kAP%f>;<vC3o-nm#WKnK+@s8*>qou$DN
zU^%W#>gBwA8-p&yJ4NJhM-M~EaYkYEq_t*w%meyfp*CFu3VZA<j6HUVx$OArtLa>!
z_*QYlvIaPPzmCN|g}RKzKEN_EOHgkG!Qm`S%9Wz1$Pd85w5#G5i6jYvRir+ZZ^636
zfQvs77Zin0>C(;{#f;yS@4|S{DV}sQoat;NLoOLi8n>KGQ@MsV=a<9H;Vg}?2tv{K
pPU-YBV^2aoHuW!xkc%Bw;hefX_f~nR4>6+@mPePi;I~Fy>pwC{gEIgC

literal 0
HcmV?d00001

diff --git a/scripts/model/__pycache__/text2embedding_GAN_model.cpython-37.pyc b/scripts/model/__pycache__/text2embedding_GAN_model.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d28a8484af96ee9fad76a93b9110a6372b950bcd
GIT binary patch
literal 17664
zcmeHPS&$^xRn4WAthKwVmsuo@O0q3aNuwD_CS=DTw2Ve05g|7UX)N1qDT<oP?Cz?m
zrRQbVEaht3Xe4YfgJm!V+Zd`bo3Swl0%HR~;J|?G2qFl6h{)gr4A`MRzWDGDoO54h
zZBsqtHTZFL)T_+<)_m{2d(XZ1WuGb(vIai4KeJrxZW+dZ@FxBwaPcU<{z=m?T*Eb6
z#)kei+t!9<>TA1gS%zt}6C25hlG;d5l(t$<JF}6A@@`x+yODM6jog!ln{bma8*b9G
zn)$2tMgf$Rn+7EvQHr2AZU&T0L|FhO>*heoMU+LfwFGM3Er42xs3lN}?gA(a5oH;a
zMRy65rHHZuO37UYWjUg(g0kYSg0dP>)<9Wv4}fwYq8tF_pnC|ELlNbmYd&XG4!_13
zU9Xs-^L$M;YUf(snL0`FhZD_@e-w#lY?!XGVY$xxjZIS~BC}z;8JPfcBk_JCVQeJP
zY&Oh3(W!S`Pd&T7{u-9Z_!b*6jDLG%y`I*W27mdr@%6ueBrpPVU|h9^)*e=Efa|I~
zu#qMP7SiM(fiyKpB25ocu5mu2)F9P#T=Ni0pSS!oLFOe>C>AIdDDNO8$ToA$d{7t`
z2dO>NwcW%uwztqJ2H9Z_{aAZ)k8+FX$!0&zw3~EO*VxyR%Q2LKLUVbzvS+x~`Goot
zmpLWZz{9536`xRXs?K00Q=7{3`rg@!6{cO)-Rbs%uyC&7Hk4Nn8r@E<Rap&_es9N9
ziq9T8)oP>D2&&aE+vx1{f~w!>dtq+7;ksT&-(<b^InUMCPN&+cUGbE!ID2SmySC$n
z#m9W#Q*8eUrMfD#I-M}{1Z%mC&Q`dvsk-gzj`9LkYjiv}Oa@(5-wtzUTHRW3*6aAF
zlaXGey#3^}Zwu`$)eBeMZgsO&3j&N~r>1IcFF@a6qTX|BVczvFHtJrr-tBdQN;*vV
zUTah3I2{Ehc_u|BGS*cLpc;*!I_eB_mZ?_jt(xywtDiQ$eeFc6Td%e5_3Nq@)VELU
zG`nr@O09oFb^V4{-SYgPhjz|+o%(jWrp}+Jx4L+w6OXrgKE{T}J<-P1iO0M5KDXm}
z_3i4DQI#jXj)yieNhi>FV<+%Ww85pV6M=U*cwjW3>XVPHS0&^4&J{%|w1TgH1c_l5
z%~bqnIc5oehpnP@&`KevzdUC9<MFE2<2~e{dJ5mi@%3Lt0yZ<$B2qJAd1Gh~%sq2A
z5hR+)W~!MUI$%by=PuZ7XoB_aJ?jN)H??bzN@U2?LgS_|UGo(yPSJAq6>HPRW|+z$
zo#mKwz}QU<QUiO_0xMg!4}z7m<>+%#*1H@rrFX$UIm%**SYnjBn`L3b^3hJOUBw#g
z96eh(c8-fDEPM1+rE;tc4huZD?0R+O)qJc{>q`0Dm2%7LYz5o?(ao+Z@6_t&kx?0S
zGc|&!<n$m~4L^_pco%wL2rp`HwgtVJrmJ$(HEON0AJhVmZGh*;^{`cXsx$M*C{f0K
z#wPVKE%8&IJ$CBY#BM$MRAu}GoG5U1P}T*cGxi0+c(ifcJFYWzn`O=cx)|49ujJGr
zaEdyJBrILO57ioT$)0M7MUuUs(eguEVf(LeI<aRVMj$$(`3aXn0I`04f++j6;;38M
zL@b1`-eEzneKhl7N<T#C=&b@VlF}`O>1bBMqQ0m`Rn+Yqnj~d5p2j0yswuaE8-Edr
zkx!XfD~oNkXs(!VG5e*7Efg2FMYyUYibGeeS5lOj9f%^z%qq8Puf^+pwR)jfYv~->
zxawZi2@8>Use7$fwK^VC#Z=tXs?3D4co|=x`_(avt3s*cX7phZe*9xdVr?k8&~&Yr
z9q2+RmzRwVKm<31-=t{76qQ7ncq|AybxZ1NrAJx77X3{m^ORqZ7-HWyZP#3Zf)gc`
z0xcal!wlub9#v%4wSwF*&pkaXxHhS`7`sqt&1|#ST!8YjsPG7KFplK<YaGHBzWDt2
zBci&XsS<6k)(LGta4U-{gDmv`6Usg<#D%+3PeXUqRG5G|fO>f*RtLg>VM5h9ZfIUa
zW3^x#y5NEekW!f?+*(iz)19tr*ILzzEu1C|qd?$Zj2B|WDAMy|=-sCZ(NcE>OtNP7
z3!KQYbdA@93Wf~wIKKY#NCwsrz!1_DGk`0La&=%rirTJ)J9|husbwo5M|6#OgA7iJ
z%*9+claQR2YhSa5_}wF9O4RamWaK>g0%dox(e3$@TVLc%u<cO{5+5C#5F=Wv-#r2f
zKE%pXV_6W%LI^oltVSDdTBJppBN;`lUye4^WLK0}lTB-3HxhmnTB7l(D$7)wTMchg
z;+;Bnc08~#{DCI7-A22TRzHCS2vf@2@oGVsfy(drJ6+!cTB&!H7n*0nwBOwf+O^AJ
zrq}Tyy8)R{j}k}`YlaIm4;rRXm|`Vd2(}x3b*x@O^Qo{vj;zKb9q(|xwJ8LC5?{?s
znXH)wBjxee&kHj}a;6^h68VVQT6k##7Xh`eF|_at4Lz{7z%P5?2jMPzn25LsK(Uz|
zrUq6s4Oj~8O8y#V$T`sI#xM(Q4h;^Dw1>HvpVrlz(1Vt61d^3}w4H)lXIs4)l+do%
z0>#0olT1!Ad6>xplP8gc_PKT&a~e%#ay*ghDQ2f+YOd(|(=u6|MuB(Y>x;c*-imqQ
zx1W`{kp;ub97hRtg2{bML^5z*rMkMGSEF|0?OZ06g5*(reabt>%sbAa(CWAyePWE{
zv+-E6x;c@pl%=#Y(yk-2H4lp;`?5J9T?^sbsoJ(%>(qK2^J%@epkifKzxDBt@$;oe
z&aBtDs{ByIQDSh#eFxJ%BY$ZS0Ar~;Aw4O5V^d_t2m#$0q>*N@@{amxF(@E!L{{f<
zR~RHkN*9rzQ5PjY4XLb45c@gmeFJOPBv%28Ia2PU1IM-Aj60NH!nZVV*e0%)@k@hd
zMa&p;kEY3Lur{!+QWqRR&Ow+k7WgdbCa=+8IwbXB*2tI+yD89a32trPHar4jCVf8f
zs;LrU;@lz{!qNQr8fWS(Hgp;gpjE4TSW`~e8JW_`?OI7Z3(8nYgvm~~;a>@J5eA^>
z7xW)yB15RU6CQ9Iz}_$-s*n!K^Sut>7sO0h@F6VDL$V3-7C56yY@ZM(OrO@Klr|*N
zo$95oa{X{At~3JgVfHzJI)rxWK~~M3A`)GP?z!I$6VG~2zb(wt5NU}r(~6GUOdVh%
zNXy0(Xy7H#>M%2IK~l-+jU^V27&mWcmRLCN;1&0gqQnT7bn`InVv{c371K)Y918o~
z)`nHIQ)U9PX=GsFzG&uQS}aPAgB-_Z3CCOnW?Hd}%rpD9&*%QJlB8VWhq|9EPgx@N
z%;ke%I0&~r<FxT&b{FQuz^qyq@^9C0!CF59Wi2u&J7N446s8#VtY6(yHMe5Pq=ZQ+
zF_r=%d&8VIhpQCOA7Zz%)j8@r_Zjs>g*qiMKb-54?Q|a=hr^z>otn0K2r?Qve$N~x
z`5;<n0~6U|Kc%@Bcg@ZUa%S@>-Lc0gvqeI~CIZH?hd@_a((k4R>G*CpEr`n;q&I=B
zAkVpL3+itMz<uX4{xB%Q4Pguy1__L0QKWxHYB+<;xW;g4;0#O5#i$%y>yl^-x<>Xp
zth(%OE?CA6Ed?vsxlVKSDv%tm)6g-|^&pEY6Alx|3F#|~7S;w?w3;1c2f0CN(*|yF
zY9>~Hq~aqq(e85ZX(pY95;{j4e`1)_drD!mmV-vygBj!POaO{^i8d}@E<^jjINJx!
zpXxk-G;0^#035wbgCq11_oN#X`|`RTXQ0FMgUw>3ksbFgK9v6*c5$)cV%A(ZKPF7X
zlXGxX1oR?EHkzCDZs46PU-HVT*C`Xwl;OjWX$A5s*Yqrlfq13cgQ3~&wcK*g_mBnI
z<f(G2+ua$pM!*(y%R8!z@%U)ZD_`2~w$NZ?_Kix8amH)^i;rNB^)Lf_Q8Pi9n^>;O
zfwA?e?!Xq*@=XMC;$p+QBsfn*vIq(FQ!GQ_qD|6uUQrq<ntJL*CPXW$#^f0!Vbb-^
z^|l~8;I|1ho}okp2ohs7Kgpn~$!daUauf3(7N`p%e-docJVj0QlGoVU4*W1(?{?_k
zQ4Myt5bKMG8m6?a63jUxy2N(;Wz-T-Vl2>5O__%wkXP(Q*dlp*5%+nBvPEmr%!0<d
zJgxvRt^Sey#9L$mC-hQ0hYrq&RXZPJZSk1}Xx#u~Q4e!YeQIsM8MAoINhnh5_>MmF
zQC3D!7A;qzEN!_a;IJm%nnYQ-FcUe@x(ZM#7R~%6oU-xt&mfrxr3Pk`U<B|NX)*vr
z1_i*WNyrS5-E`=8b<Jx6rUZys>p-11SG1L^h9W)mtf9zQC5tu|j6(eZZ>7c%@Pl^G
z!8xm8Vnu^x7`1B453>=&L6D7=`Fzd&bv`inf;La7ofD%!3!aS8Y+bL2p}|BMbFcV>
z57cK=!;zg1Cx@jjfv&jz=wA$@5x^GJDp~)+2B_8zv%fm-a8id-Z}LO<HqxO)90Pds
z&pWCJ{F&<9$`Po8feDAXg=N74(aLeHYsiI0NzhA3T)CHkHnBCd#=ss!LzqIDYT4z2
zm}KX3{E<x;ntj#5t!&}2I4z)67dhdIMoO0&!FH86(yLyCt8nI##hruFk*$I`r|-z>
zFHJmpw7;zHdsmUAzQ)ctYw!pa99xk#e^1?ooRQ^{l05}HA$v+fMicRobht9(aTd%w
zHcw<5qdnPEl!t+FHDV9!7498-1x}tI!3E)6M5VjJyhezh3eq$w25BrH_Z;^Z%@nf<
zcu>tE$EYIq(S+lunwR}%ljX=G>NX@;Sqkn-LGLax_sFN(8^m#`i%7zBw3IS2!m%L&
z6q}3__+btq0|0GW{(k82r;2?Qt`hX4o~O%2zQV}=s4lbZpJDP|CNHp;g|UR!L68Xo
z;}yOhY~;?H2H5|F27EH5VP+F~%YuY1LSh3-W5f1uo3{|-{Td6o1ytvv$bLP7-3tr}
z?-K1e*lzad+795$kHy+yU`ZfNAUsgu&4<qk%h1dKnI(g)dM3!h?n)wz2uz0^R^WoU
z#-1H{^AW0)dc{`|&Vnttpgw@s7Y7dNX9gKJ0Tytizo}j}(ch>aB0yl>huD&c+QSAA
zWmf$&`WD|J=!wG!x|zo1hl3@c=25w;7EFSpyI>IDXrN1A(i9MF@&$Xhu<Ps=2e54i
zd9+;E&0hHC+l@goo(0H<)I?BB-GhmHjU3dkFg|pA?(&~FA$c+HR7ztU5hh_rc762{
zn$S!^_mUQm6!xJNbi<SmMX46E+Dy1+>OD+^ai7Okn5B;r0kp<>_@Z#V<y96m(tw>?
zB`?tb9eb|<OUSw+pX4bt_!x<=$Q@pcG)V+<LmMzMEW|_+YoU1|G=p$dQlpO0oNz{K
znN9)sKlu8`kr)>6guNyPU&72M=_D8PFHtrf<RySY4hrUQl8)8CbuKT*PIUUs_xIx5
z09@>1ApglB5k5c+E-6x`gdpVsKmLy)iG|>e9OIEQ2VxQ-<1`p16Bb4yyt-BEOu-Av
zUv}YB<jlYea4f>a6lWuFF@)#~5IJ+<3BRiS=9wS`sx?E+k_w*3PKx{^xgZ6R2(ik&
z4ZklzOF_hALi<G+*3B}^6~$mF6e`6T2ok0mT1=&)_-(2y_u8rPAgY8m;BJ^`R5$Bk
z0z8itVOFG1KX>-6>UC^1^5Oj&uGPnwe4O=CI;1uqk=zNvq@yY(Jp=f&*5K(Znf<k?
z$sNnLc;WaMl>Lw6>%Rrf1dwn|^C}zy2%{ef;4$a@CyY<R(**0*8Y0$#V0r?%k0KWy
zb4?kQFmVt2SLpafxd7!-kefpP1bk0%9_l2bR_URST1IUfag69bu4fs$w)pW83+1}G
zj!0k&k&^MfC96$~9=5fcVTp*6q5n)MY+34{1sgIvYC-PcX=}>C03aa8%>n^2{+g%%
z0J1r`o08U&(hhUP%SR6Z7&+I^yy<iq>J9PE*EqfXd%ep$2>q8w<T5c%96St;l{?31
zHzP#4)eXvHZ+ibtI;x6jq2Qr1quS+jUis+#$I9!Ib#%70+J8hF$n?E2&M4BS452sE
z22DgCN?LcSOX&s*7!cu?9;MpGBW%5U<#SK|d;jG}V!T%jZFu6fgcZ;y0{Ik*s`E^q
zKoVwK@aJ8Do3GVVzfS5WnT+8fxm^7Qk~s2cHbS$bXmN$sIX^Tnt6$-@*`0?1kCWGl
zxeVJfZ-pp40zr7xS^>YqAOp{rpyrE`!X%pn#}qHeXI=i>095v}0X~llu!n??<QGgc
zVmzpOVLUqW$u^VsR)B2)h7czd7?_NwDGVuLyMRK*lRDy<8nqyIU|d*5pn>sVibag)
zOfVi>OG-Oryo|WCG~-RF+_|fk)#mzWh193;oZr#XsNZJE-(kY#QNPRN_n2^n)bBHy
zM(W{#Xojm>jf*%sq7gmZ5ofq=bg~-zt<)Q)Wz|0I)X|Q7_)E<sE4PM=K!DlHgjC|!
zyb3WUCd;S_?Il)u4qyK;k}VW|u|zLCY#_=YRQ#}Y0hW$D7J#vbQt)s0+FlD9z&d_k
zXzt8B^0%`O@iaoc+unKjZG1|0lQQ}bOh1pDm6<q7bNnGW?&3d;i?}+vu>D?Jxo81<
zZjSL6F3z@dHtrIZ;E@}5i48}1F*ls^X=eNplN&&Sk0QU!{<v*`MDor-r>ID+<2(A;
z(be}7FJC&%I6<S`=+tmjNgx52j354vAi)l9_J;&P65%)#wvasm1{|SUGt^9wgl&Y-
z+=S}Su_OS34Ml^!JF=CMBQT(T75yttgAR;D_7WwC_C`>DZ=~u~zu)l2`d@v51t=A9
zk_w7n{U+~U$3!GzLa#Y1hpfy<$`Cj6_5KEzf*=>32<m;&`YUGt@O5)|z2@gcjs)K~
zaB~yQzn?xP`l3EgMq+XD-jWG7h|Biy2cgJi5s$mvQzS}x)YiIqd}jq2EQ=P2Jd#JD
zYw*~f=wet>aoxD@Aj8uTw5D`laemY`MM1j1I#$N_j1{xm!}IsoX7%-~Viuh^QibZb
zP-#{b&Xci!%yKJC)|qgz5wUG-wYv>huX;+aIt6QJ)+a=*NZDxtD?UMw3s31_*hjGF
zRDAr46`#KDAG)3mV)cjm^5HMH)#LbzPwR&!MAaXl3kcND;75Iy$;@;8A!t|lT#@*E
z@cPluioW?GRDF)E?=M0{%3Y3og4De8<~`jg5PmbENty5tcE>FTp_y}*6=`*RN1s^6
zeYXd04?ZzEa0?&Hgya;l^ex@~zQ=FBiu}1kluo1n=i|2$zlJ=({?<|RWr<d=;6U!`
za4kBJ`xQBmyCRm_%z<3R>4w>+eOr{&2y4(`E$6$mkk0XzK4N`rm7-J^7azhRZ<m-z
zJbIZ|vX57JMaVG@9ZSf0qqsS_2%e6j5NPNqa&*o`dM%JDU5|Uc4Yc`^ewX}HMz;1j
zN0H*GgQz8F031cVV?Tx%tALo-I47JW+5E5K0#2Zr;3|*3Kyd%)U@9YR`26T#>cqiR
z9U7E<AlpK=%AMdN4QO7^t2Zzqo5yb6yGVE=lcP*3OzviK50hg|W)uTmV;>>^b3a~p
zZZtmThEqS2d#|fvQx69j$nfe5Nc1Vv{Xy^zpzoid`hcTgWAKI@P^8S4;e5u}8|L0D
zByeP&_YSAs<;Z$h?Gqdz_?RCYaLvg8fos7VtwRS464*0Af`kqd06PI|;l!yfEx!VL
zb5Tw{&J}6_n9rydL0iD@f_UK;akqrKCH#(q3(57ytC%ziH^-tll5!GeDBUE?kdML)
z;-VeL8peP+bT;cz1UsW`O-33>lsoM<ejg@FhZbJKRm_Q(N7VQ7>IEiOnY_$IhBr5`
z@L6(!fG;zfq%+nM=(T7aHrJx?f@AjIc;jUqOJjm?e#qUF$RdrK06_q-KjcsY0s`+r
zKp-(l>tI9_94N?PlOhhX7zjJ0;lD@$+^2)3W@)%gD~aXh?Hm3gNC7vh(Of|Q!j{0q
zDnk`i(Qw42CG3F%Z9Hy<_TjUbtt-9dTpWcNjUqD|ksieXwkV^}EgeNm2uwsF6lZvl
z<)No^?z70v#ksi7k6JUEhY%U#Xj9F@46oq8-NYTdF9o*<QzH|wB*Py!E18lf`yzT-
z7-XXH--2iTCj@A29WdmBP>j2XkPX^Ih-m+iO%6wPW<xeOO86>HS)p}otvHY)L7PW|
z+pY`R+`U`CigDDr0bUP@f;Pq7%<jUVh@edYL7U=k?p5o;eKTR3(rnnKe=y#7I@VE^
z6U8k*Xm8{rfFWCQ<RRQhY}C}beqMInadBYY-(Q_>Bqrx-n{twcZo-X})OcWHbnaJo
z_`syG)n6&&By9PPIJBaRs8_ICzMIg}k#T5A{VeM48(32BXYL1>w3xJ+2qgR>ubyD?
zDJEY+f>*9~aO4@T)^0n*)9|Rf`U_@c<>f2*j&`nu$xXg+rT&uiG)~a*<t=Rw3jBMX
zt^FL6pGShT_>?c|gG_#b$u%ZjCZZ;v;?*xA(MK;m^&t`n;natj{1OxK*nNao`<ann
zrX-`FlYkQpNuDr7z{xH##b3#~3*I_8{K7k2kt(T$5EQ?>OW=r%=|sV)6w1ZHDIABn
z>n3tAvPk9ij_*k^|2c;LnF+<bsz~_(DCRroj(p0<+DAEm69bs9oX3%S*PK$!R;-wL
zfLxUAL_S}+ly?#{0S=fmH~PYX(u7wy%FPC8I!O59h_o%g+IzKcP9e@gJqxB5B)zD1
zD)<G^a8$X<w<EP;FX6YulK4f)nl2UX5u4QQqVcB;FYO{*zOk-XvbO|79>DyC%dzl~
z^8<Jj5r+qK?Ds)tr=jAxKytw%VOCxxY;3~c_QO#7NvLH1PIMigCxE{tfnWp9fEA(o
zr<DIaH)oye0RRO|3SR*L-vmeR2LK@C<=l95W&m+1P@r0HZy`>Vh8nb+c!x!NZFoBn
zs*^#B)C3jEHljK~M!ghd0RocEoT%a~Z0dA$Z3D|`HCUKXmvRkdi5H+nSr*k_2R7hB
ztnM7DyFq$42c@;xTvGpvp7T&i8PqYOIvD{3SvNDtjrz?(g+^mwO*!342m*B`PnL%g
zWxWS5UeMyQQ1C{x#IfUa1XZ?-k7p$SpwJ_po!85(LD(WdLbtRt*%D^4B+pP76!^;T
zZb5qc2+DvG&&Y6fP=H$3d7nofPDD)PNgJ%aN5i=&Eqnu|7DlB22L?r5V^G8vTAEP4
zBYhgK#OzO2ir@MfQ)N@=(**vcs?<tx@$|{~O(PAC`-`VX1g$;$D`((hZ^hK;Wu^Y2
zrs6$Z-<Pe_&j)ymwj8}-65na>Wj}TH3IFWLWBrvW4_b8QwX&x5t?cWsfU&jnggYFn
zvDb;-iq@L+@0kDhOlCChI%zXH@)Bvp%IY7Oe3{8JO#T!}bg*Nt^870DH%5X*W;VhS
zUz^I2*^P3hRcp8qgOF(Q!vyC?1?+x~;d%$9##v;Fip}yQ2SW3fs8>jiwS=7ONG70o
zWA&P~7SmeM;w1o0#bGy|T4F$x>55jtX8-Q*K+b&M3ifX?_yiv#vV3n<r2GIB?7_LC
zN0fG2t<q;_<Lf_-WKykC2?bDvP)~?K0N_GWMMp^!0pbI5=r|td;v_?K+7d_K^l`WT
z;#=`@T?<ijz8Z|zmG6L&$bPk(P%K5FiZC;>%ViR9+Gon=N}%}tnwN@Gokv5^GcYF)
z@+=VCdi%}Q^?HN`CA5duBV034)aJUdXMDV59<-dpzVGH>@I{M#9B!C;TVBpCRwsh$
z-#do;M|5yug2gh5l*%EI7@;Hov*38ABWC}`a(sJyLWq1F<@NCc25a&9{$GKxQPf3i
zy`alCRyehW`f(<dGGPj<jyH(4L-+xbpJmcxB8YsKS082ac_#Gu2)dNTmoZN4d6~X4
ziiFNXm>V?wFqgrncMI>o0ZcpC#R98MC!MfL_m=J`ZI=$?G}l9=x0GIAI#GIOsr`RT
CIPrl1

literal 0
HcmV?d00001

diff --git a/scripts/model/__pycache__/text2embedding_model.cpython-37.pyc b/scripts/model/__pycache__/text2embedding_model.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..73405b38ab6ba2bf2ccb6de2a5f260d2b8f4a845
GIT binary patch
literal 14326
zcmc&*TZ|*wS+1My^4;xr_w?L$y%)k5&CVp7t0to?GqXE8kWCZqOtQhd!ST4NyX|({
zJ*UcZY1v}QOo*arU=tvO0KyGHC<r15BHSVf2zW*jFO&pYiH9%}50D2WKmxw+KjlmJ
zdM1(bK(*@ksdKAS=bZohZ>KNj^BDuz=D+{<^>o89{+SPx%R%CYaR*mS!|)8x>>73c
zT3vI;s#~Vcvv-`j6Q>h9j%Ap}PO_egbKJTsEz|Whe(kP3Epxh=ooqe3ldI=YH&M^~
z#<<`co#ME(Z>Z2q_AD>8Z`KPh8lLN=?-^d&cREXV?RptG883^RY@D-<oSc_OPCm}5
zAgACJkyDIwR`8BhuT(#R-!+sjd1aKA<I<xC-u|5fZ(sH*s9A|?9z)v|ZxuPKan5n?
zNc{wI*Sw?1JsRhpM9wkqIC751IgfefRik#|0q1wUW=8Huo2uQs-u02rYz%s>u-)%9
zyEQ9Xd7;<pd%k*keZBEYJKSnq7<lb|WIw;Yet9q18hFifZ;$NO?x4lKCKuO>S8xi%
zH|nNm)Gg2bw6S4&j+eNXuG?PPOM0n$X5IO;;TZJ<#*>LMv$s9KN*mv1Cx-FnbL*{?
zP8$5>ZQ~BU4iXxnIW+ECV{0D^K18}<4{cCqXn`h%4rp?i08I^(p0S-)YMAV}o_Q3t
zx2@oEnEs?GITms(<a~@dVWyMq<ih;8Fih^7p6xmJ*<Z0&2s7g>-ec{{GwPM_CY$f+
zq`ZWeyw7(nc^pGI%y*W@m3?g0ZAX30V@cUF@Uf|N#V=GY$Yo4>azpw4o_`H<p7K<G
zw?7D@{PniiR{mr)R-!~O*!7j-_eXA{(eAawMkC6!d%J_M5w!RGD7)46Jin(OGXBnW
z-_vQg*XTCy_$p9YwoU|F&0Rk#TnGYRakXDis;?re*Nf6Gu$9;DZAQfn)!%9CDnC@s
zcF*^sMA%oYttflB+i!-~{9b@IX?cs(w_kkuBaywS2GNSwZ)|j%VTiHpHdS-S5Ap8E
z;f{%No`195@*A!GpcmFskrVjc4OQfHERmE*$|N$@6%3#ekDxJq8RjhAXtcV`AZRqc
zV*J-<&UX8)X7`z(rJ7-D>+Ei)zvJI&?wwWrpzSv{{U98mpX+|FwYAe!+h<$dK0fK}
zrS2fW*zmb$caS=JssGH?UEgnQHC~LHyy*9Q^odD2i_Y7-VQ_W_JkdQH`nSVprvqw8
z=9%3)s(_($_`zc!hFLI^lRwKf%lJEH6|6NYiITnL*_J;uS<%*nf4Di4xG&)j-T=WC
z!}Nff@pd=H_R!onM^5N;5}jlxHFmKPu{B57YGV`I-rl!fvqs60J*|=EhL$`sgz1@|
zvL+NgXFg?Z*kFVMCCq0%mK-rg$zgJ6Z&=vIR`avi#+hn-os+e$#v9VV5uBSAvBm^u
zs{M_sY{2TN-e9MJ`QJTtt#<l47f!b8sf)GR=_+<t=zCSqZz;bSV1>GOs@LySyMAvo
z+zL)@^i_4Y+1f@yZTgx6BZzAr97MMrgfamC#sHhakNZ2^gJz_IP04WWX15xI&Cq8b
z*zdD;V3UFB9r|R{sDhWlpaG_3^3~T)Upzg>si&^gW?#UG!mbXhx`Mon?*as$YM=4X
z=tA9Rm2-es%v!J4vg#yuiaG%jmEX#P>KIGcNLE-SF$mk;AhH!$zryJRzXC!48}a-^
z%g8`xYkMwJc(mrKD!Z5f5Y{`&>$Q((K1%A3h+NGkfRLo_DN4n&5*2i!5jRm!acDxG
zX6Gq<;;p9geh?1<k-^9%&5V@+8<ot8`95>6JjX(D<(e(Ps?w+q0k>XD5-)pzA>w6&
ztUBm2ooh6144PeCB8;ovfi_V-hL)D!?KT>-G1W}<B+3<sCOAyM3dpanSy+*r79OUT
z9jAiRNK7Q5$U)Py?zxbI5G?nMI;5hP#BV~RVUhwOaxR2nuVuwOuqR>xe}fH>MS?GM
z#^C#o?U@w_HxW=t<fTG)oF-1}Q#@uoE6k2_<mqwVvzdF`7(tkIGMz%F2*G7h*imo7
zI1=j*ID}2yxc==q5nGT_&W_*gMRpK+wUWxCNWF)I*rx$pc31Kx$c?6o9EbvlmJd%v
zfo#CYQO%wgnK#i{Gu(n4xS>K&3NpuQhRrC|>#Ln+w}JJnPC7<~(7QQVh^Lq>tHC!v
zhdaoDXw<D>ldPJ1c~0aET_<ZoF`B@A33u>OkfAk(ZV5ES3?O9@SBEB0)b=bq*<<3Q
z##X2saT@am3{D8<Vy-(0AgATo_pLF0_o-(&P3V<x!N^7U0zr4P-5&(=tS>kdZux{k
zYGSA7U_?vwADSu&0bu3I3=3i`1jw;rBWAdRkQNtCF^Uqu8Z*@VtB6?hU2CwLf*(Ok
zq&>xCl>&3K?aw3L#nabj1DmNnkm9!2-l?V31uQ_6RQ|5t45Ku}eJ|MU2R<~FR$uv%
zc{xf2{f%&^c{@rEdI7K->Jr*j(N2-OqT-<sjZ&ygSc!_^Ry%0S#7kseiSpc$jmb!7
z9IiJLLEuH)dNZXnW(FH6hrhj?Y^E4zT9duReMD(3d#R2@Na<^gE&M`453Nn?mwoI9
z*<JS7iFXf_#7<(I99o?eR8mM+?yqs0I|nk|7-t~OA;GaD?QwRppVrlT@dk}>R3dA+
zn4Q9AZ%e%&IZ(-)q2dRq50G3Yxk$21@<EWuzP_`AIgKYWF`G#B5{r{EHFtFTr`Sq}
zu-X+ozJfatlg&JVdEvd6k-3ot!^%9%M(-qf7l|MP=T)(}dN;|m-w8WQ_XH9SJ0RY<
zX3lj>0xm5)PA_6Eui)(<6J+(Wf~~|-+88O%6>QDH+Q`7xn4xPvTD{oZ@|wNofMb4M
z^A<#`%<8wl^h><{<hjf1Es79c{L~a~8+Y&$ru`89Qr(A+rJe*@5PfNe(DX=%=|PQ&
z21p9j9lD@tth}o}A;yHPG4gVF%5#;GE}%TEZc4dJR2DR+?&qr4hc@sUMhA2;_LEj$
z7^Xe@y?in|OP;fBsjr3Qp?#M<EsNPv=_Ngmp@RAq^{q+kS*;%Q#!(vYTfx0LOmp8M
zbp*dOhSngVQuob$e&<m<AA`w5+)H}yeSYV0X@zf=ah^bplbrSY9KmDq=EuWo=ZW!?
zFox3G&Kst>BI9~oq)b-t=}k;7t~Z~1KxT@ZYii(Mqj{8qTG4H`V4#XkAghG*3#e6l
zN{b&N_)PpniC(`Q+=;TWc0r*cHbs<<&8x<4bi`{zqlanI0PZM1@Ox0~0ANu*01Ryd
zGexHtMM<r)e=1K=>UnLTY4ar2YuxH9FNl^Vjix3+RBR5y{sfi)I9f&uho<n@7y#fh
zS4GjHVyVut`|L&P!RwHG@9-k$W&fp*L>ZbuT@j_~IL9H{osDmcQdc!9svl*(sD7f{
z(e_Y}gVZv@+7Q8NGtgF+7+9~df>>D}XX+<N;CL~jCH*3pmlLyc!kse&ATxggcfhwA
zR>4l14lr_R=-|0v=3wHKq{Kyu>kvg<D~Vh%wdGiB;i5*xE|>+@HTNE0%#}0YO}yhp
zorJ7-3Afl;x7V<-0lNFf^Tx+BBbYTqvtixHy`t3^YyC2m2$}iJoI#pbn0(mSL1R-j
zy_%(`Dj_vy=p-^B%4!q6frab+GG8kY#8E%_kWtSyXi-A&qM1uvlY#Hzb2#jSHeJV7
z=OH_wd+eLz1V2RU62Jgk?7Gx20r_C|DkwQz4t@hdd;#s#(Bo*yQGNh_AWKAsXgM?j
z7Ez+`X_N|c<2<BPY6F@dp5317CD6tk0jtJ^ePdk24R4sB&60X^2z_-s9egt^!*ybe
zm%;VsxFS+3BYnBU%%m@rub`JzmSWWGQP%_tLPomOoM4S+N4f9_P~q8d4QSzZj^2d^
zigXIHF-{M&NSSc*WOZ+&EP6OL%%ayU@;FA0mE+7GdM*sJ!`v{rVM8Btn-+G@RCS59
zq@6xFUvK(@Dv~bIMy(iJt$|X2ifY*2@nN3%yK{=ttD<F8Z&#t!eEje`Xx23rDSn6T
zy+<g0uWF?aZ<xGEcPMuATi!Uqp6^Nbo7$(Z@}q+9@fA1Q9%jvhLubzPJ-0wVj1|~e
z)tt`FdO!5fRd4xKHRx5Ts8!*Yk!giSS#9cB7Q^>We*gozKj?Z@0B5z?V^_YacKiL^
zX?N5_!+v#F^)a9T9s1Q<Tm3G2jP2QJ&6xuA;D2$6QnenXVF>G85M}4qt9E>5R;wQX
z|7yS#dEne^`?o}a)xybRF`-6XB@s1DF!^OXMoG^Po2@NH5P*~#?7(A0+*K8l>m=0A
z)E3D#kSO8#*9V(XVi#VjP-}2)<kyNCt#b2QR|RY)8eDd6WuiP~RO~&26`muAHE#Ls
z&8;wqQmuZE&Mc+SxSob`+h}e1t?j5d5ovLSD5*D+R=CY(WL`gnl43sp8oCM2gBV5H
z>Phn$@V=5H!k5jQUA7$CNtPfJa>yxJB{SocP>cF?#-@}&p0)Sb!%$T_1Zs|6Kq{h_
z#mHZ*a_*vBM)fF_lga}ZA)wyjnlK;FxsGCP3-|O|taPSD>7*e~rBfU74gv#IKj)Rs
zY?O|DiG2mlav~A<OYPFe9Z)wuq*>C<3>%_DEeo0=Xd(nQLxVFr&?h^zAs{O{E}hVx
zd0(_k(W<O<sNv#QeWb*Ph9Cy~C-H8rLCtgnQ42)1Vi$1TYSjl$;r?}as<qx+i>-%k
zR>{QM5db(-Di&)H3_t|btF*aJErEK3{<GlOnNHVR6#yy%R02efQXQcjrTCBi7I6Al
z>K^hHHvryWW%{Wpd73E}{Ri%VJsIZS%It;nvYV{Ri~9dmc2idyBC@gVssc3UWbXt=
zpw>bY!2=7cg7u-*=UMkDe<{!5HHA%(*mod>Y;9^xl-PuxE5U#%R>=5c8#gld)E&H1
z7(6N*1a`HDlBl9Jwp)zTQ2X^8H{m)x^vSfq{u6g>yJ61hOSATt<~}`UFzfqfEAFU&
z;mbH{@Q~$QTQToi3uJmtx$14mnwoM+;VP&w!c`0#!A&*|7Nw%fBKP}xiww+57#Y7d
z?%+Ehp>a3fAK(?N3v8ql9(l`xWkjwbKk=9RWU2EZc`9+&99g}bl$f1l1%9$H#Wh3@
zco6l;cS-omXj{R0623(HAwof77iGho`eK-WM4N*8R#;>-47v_`1xHtmumrn`;i9w$
zo3636sAs`~GTJPo-ZJ(Ztvj=`g5MS4|IE7jaeO%}C`(JGC-8*yRM&q%>}ewa2S1<K
z0`*+1O$^}b4(e&Wqrl4bj7I(j(iC9~;6#bb&2He=RyBN(38OVxm06eu^|gtmjV~J#
zEIK-G72UjnIMB{6WPUFUwC9j)rtY%0RJ_e)(+D@fZ^H;A+GvTzy%Vjx_~C2soO@>z
zA+wvl;>W4al5o342!U-d5*0*Q1f^h5(Z3W#E`Q{+R%ig?jlQlId!%D8w<5TyUm*E8
zk|w(>&ZMdiv(05NA{K~bUkrXpAXxAL^kRt6gJI;jFqU#w&aof}88w1|1S3hF#9XqF
zu0T*qi9<6gY3`j|#0N9H<Ye-qJC;0g33u>1@D8>Eyq(a>t&ur&rfDZcY>9giw{y%u
zNkV%h7+r!76!E1_8X7kxPe$DqPqL%^$YVD&I|X_#Iz@P~u;)7^3A6GS!Bw$e+;B<#
zjt8_JmubaVuqj5FVFoq?{1V_pgj2<8!P7Bg1HOk*rgu7AQGY8fDD`a5*mqztpk1##
z$wi*6^M)Drj?q>=g?Em!;VP^O1c~9vprw!-<~#>{oTaQpd7j^a8jlTiNI2!0=Z&5_
z%!7lG8s^7quqlq>J_fr2!C;1vQ6E8MwuNPeI<PJnRZF~PkBYElO2cB@8hso!#>d4n
z%5NH@CA=-3*-<$>fquReo}?dSl1r=S_TPt(5t(LrFlqmMv@G@i8ugRIe5b^k4}JgA
z`0-(BpSBXd&P#1uZ&<LbP6$1TyfW>mkg23jdG4^N`$YfgVNs8H@{1B1ec`Ky%a{|c
zRdw4yZyyG(Eprxo$0l`0m0@u@t6v>fM)^0)8()3JfV~;-x8ah+4uMW!f!L3E1E~kp
zg7#{R2A^5Tif7MC*_4m-vHqd;^=o^@OVnCndTGcxx3_vIqY57goWivyW=eUKfMcj1
zs4s9R!r2#?x<^7aD6$aq77)45tRG=k3TV~X>cja0&w&Ke6Tr3iyi3EwGV6Vw<PDOL
zq)hTTl2<{Z3<DpC$hYBR8ZejgQQc%wQV&txV$N;mBv5(NN5Sr{-}BVZBd@k1>I~gt
z4YEH&aFYO9<RUE~35!7xu6Ph01JM@LS@cs+<4f=%VcXX>!&I-(VjwC^W;lMfVHc_|
zaa5M)%}TXwsAphR0779$2le7CjqeP?$h;AmVLfvlQT4{=0Jhm8&Ey<GaSQz$7OVk#
zl^&u*(tI}Qz$T~y5Lc~of|djFZ5IGouw1JI&yqvy0paxm05ga3vbhEdFvXn#Ws3|-
zj%8d&TNw^&XRN&w3*mDTM`Y;m;g0M__^>~NPe|}%D4;N`cPZvj4oxqX@Cwd=Ox2#n
z#)S4{&-~Y!sPMw<WX?LwPlf~z+7#SVHv8v}S!h!Li5fsbP#lwyqcK~+Y{aeNqcP{g
z6#P5Pp&tp63LK7EXI|)b(w&UPejJX0awdmk^1YIdG_tPvf%B+IPd4IXpboVh(Ta+q
zpSbJo)0m?f48YZ^2m(9?li|^T_{<7JdXq5Y?;$<s=Y{)KD(Rym!h51bQ{HKx?GdI#
zr#{LQP4<}+bz-{Atfa58{0Wf7V*ATn)S1mfVa!7v75rALQeyAO!F8DlVlD+QY9(BH
z5yT8!0mK=sW!TXk!*7WcOg#u;ImiK_=D^vHv_5d5K@+}A`Hl~wy*Pv8?}z|I@Y;%f
zuAX7xPm(+}B~LR&8{q5w4pEkcGvLKsJv8!3j0+Xz;xG4<4E#wXLYzavszR3rwL8uc
zqk{MqS3AUxxlb#{(N@lb*HInwNgRbeBa1;7s*=e*!4#KRtAA2yp}j_!Ylm0lh|@PX
zeqkZZ{oc_-W1rQ7x+|>lsDRWT^4*VMW^TFP;QRRbVCJ%$RLpDPo?eUDSs!{kWW~tQ
zxA|ayIx0Lr4M8;~>^waO#ICt*Y}~;$kVD)|4yL_1=N}luZyKKfaKmH=HU$2#GVl+O
z>oZ|;?hspU;w>=M@1yrxUMuupM|zIaU;#`50(#wMEOwBmbQmMK(PGFMZkj6lDUpa@
zfor~)xBilIOfzD7Xv!*Zb&njJoVW56*M*ZLTXP+W1w4h%22Wv{#NEsQ8bh;GFv)ig
z6uboPB$PY{N^c6kI6QTT>)b<Jr~Vl4(!zVd6eYOY2Z7Y6GW`c6WOB`iuQT;iBu`Dr
z(@c>g|8Dm6;sXO;mVhmLtB1b+k$g$)BVR(Z7v2nUc0o9C=5G-Ga^{#&mgeDHr^p`v
zz`?<4pqJ$6p#jY?n<>op{qPz$-8cENnAeJKS~2ed*LN|)J&Tg5{4{->Oq+8ZICA0?
z%oZ~p+zO6SPcaS<l`kBK{gUxEC!yaK8IIQkuZ8)j`g}L$16Je2X~?5^bj$%7MHHry
zli`yN4|kH1n?01P>*kP4!vqEr`YPb5$d=Q-N|RQ|(E-<`n1%KHmo)?bDMmYEV9k9D
z_Q6M|Pe$Mz-J#&7@UD)NX&*6+^Xmvas!y=5EXi>a!R$6t69GEoZwV*<8H?h>QnJ@&
zo0GTyiHp06#FWY5g5o)yRoJ{`bMN?D7iGrq+yuNP-knQG2V}x2!&7uZXQ2=kFC6@C
zAx~I{wTFlp>@h^900Pik<ebhZfpSFD#<2e|-LP=^%b3|*JDi4`w4>#V*cwiR>zat3
zSy)&+OGy?sl7^TRd_`6V=d7@lVh2v&$a4VJFd@+>NJvbGwt_HWFCbkjXqb`Nroo|O
zyYFX8MCZ#)T_yP-$u~f7mdfuAWLXfm=x;XS!!k<NNFUUjL!+12PV|EzQgyx5Ffh7p
z-+QRa@UL7s4gekt@k~4|O7|L6bGq!;%)Mg^*6#lWuKtHZjp6F#f<etemU~FQk1`*+
z@Ug~AFRZ_K?TTg|g0q0&qjUuF+CTEG8n?;?x2%}A7_#1mk||_;H&3VbsZCE6=tBtv
ztApN7zF0u&H6&)<PSNiz$^rNW2}c%u7|_lDED}fPr0zn4$4L?^bfpG0IO{I8gtVN{
z$jIr9EWDFAp%Uxu5_)MLf!2&Y`vl%6xbvWnRGV)3HXJ)sJ%U`XH3lk8S`&DKsh=VF
zSrQ_=wz&x$QTpnSU2TN@ZNFDf`~5&CAK{2ozsUAqBw-Y1G4L?z8<bJZ?aOFWCPl!Z
z2EX^jA}f$nJcFKzJRo#&ss(z!g%v;GNV6wS4rmoPI*7wAbTZMwV~uU-6#9rD+%A~v
zlo(nl!O<_7bQ)DuqQfi~Xq38rI?N5>f`<F2ioiMb7X-|CI9qg>G{Oo1qM>`<xZ%OU
zGO->qatzc)|9JP1zclEKP9DxVTZi&P37X@$;u7-9o2I<cqH_$6xVQxXEyu%57=JFT
zAc!w%j|T+c@RIz(!F=YzJw%rn-?mBz91lJ*?HPQ+HY?(2K`A^kKC*8NOK>8Y>azYP
z7dXe3@buHShxqD@eD&K#bnxX7x`#s%meBI32%W6V5y1(S9hT3d{f+u7M(@h)%V7?)
zn7M0Vw%)g(-ST*iZaDN(8LbRgrY^nto<5IR%u+Qt2+Gur+T1>yNZ@+&Ik?o1Yah}D
zJZnb})ucJCiW^x6Yeh7k;G-Bh>`nbT$vu)k0*MkE{ErItD@-ThL&4E^^;aytOd>Mi
zSDB*jj8l^E9`$!3C+zQTONdyUNbti~S?9|nze)02BzqwBoOa~vGuG-W%%=#5EWfQj
z%?kMtekVVSok;bPr_GVcq0!jsW8PL@W!v9ola<g%92<wEfbcCiyBqRP67^-o=HZk&
zPy}yLTm3E@am+V`wioHZW%B4nR22UYJvGpQPly$egO}$N__YMT4fW!RS+(g8O5*ep
zd_K&zOHA?Pky)_!st@JqSR~28@I&Z7hxqTtC}v}vV2bE5-$W4+ZsOf3DdyB3m*uY)
z<iBLjFv`JGMG6%WebAY`o<4TJJb64b`+qA%^wx|098-S-QeW2pR&vGf;$MF>^aqM+
zkPdI$VoG+@V@O3goO5m7Y__}nZ>}i$AvxzR$I|uDvyZa+CrMr<`5ejTNj^g&@GMaL
z7ks4nk`J5Bf=+P`BKN#LMgIXnpb<O^f>T}iD`Rb#Dy1Cw;L6XGpD6#NktrY3QC*}@
Pu36=i<r9use)|6a@2T@D

literal 0
HcmV?d00001

diff --git a/scripts/model/__pycache__/vocab.cpython-37.pyc b/scripts/model/__pycache__/vocab.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..76d0a8b7b9ae2043310c7cfe8eb8a27fe43e797a
GIT binary patch
literal 3856
zcmZu!TXPf174Gi2Xf&2&V~jDEY^HVttP-#cyD2D&r8awkhuw0Lg#b|{H5sc%_Sn)$
z?iqnCMpdbWJftdQ^N=4PKc*@V`8iwtnkW83p7Na@S+HJ4p3~jax6|i*=R57s>UE#t
z`RlE3|Mb;q#{Nr<<HthfQ@q)22+1Vxv36PVHaBh27N)e?wke&qbDv2et(Qz%FGSmw
z`~hp)uQ8u(c1RvS^R*?u#EjO%TD)yZfwio*BW=uHldi11<n4+)Cp}q3&6B=dc*)vT
zS(9~?zHG=vlnZi6o<doZ=jCa+f{}H3L7tJTs5RtSXmL?Hj|QD+OL6S@+6X~4(&S->
zBt2Bp`#Ih$fGBvuLYy5EIzozqO|3#4aSmHxU9owuX=^);`mwI$D(UxQsaKy3lnmqk
zRxD+bZimm}WP2yi^pc9RI1gpg$&*1Esl%qF?JVwfm5aUfnIz3(h5oo3je2>Q5BB0T
z`xjF+^!{($>fr#rYgtD{d1t3J+#U4e!)V-6gDi=|?KsOvDh{{ew6oKX)LyI88>F#p
zecl^od90vEtB+di^TD+T!#M8jg!gAt?#F42HK9@qizmZ8YxM^*?zNth#ny+1+6%)Z
zP4X~oK$FaaVB9soal@E({dho!tV$1+G|-sxYg7_4UnzOP)keW~;c`eg9TIMbgyRz%
z<J`11Ymw>Qyn>edg6f_zyR$iV?)>IgU))9U;MWhR__8d1xB1oGW>x#-f*VOH<AYhf
zGZ>}0u3}PYSY5@$<0>W}SFymX>YM2|XeDj+;#61CkQ!N2D2f}kK!Ti8iaOP*5G!~y
zGF53Mo7eb?sPXY?X{(bc_<_xA%rZ9n1{H$eF59c9J19-P;CBohcLmi<KiQ4y%tk5s
zO_)C9=`y}zjKj`h{7s-5^Gd)fLU_|wizw7t5=$h`(AV-{+NNWkIGHZZVJz?-#%6SI
z#(f-)-X$?!DvxKZW`@lVMu(ao>@pe$LBV&~lpnD#*dLewASPmhV@yRES^KrerLugp
zX&LRbljLzfQ|B-js+0BG3xj+oR-pl@TEiF3!;?B!BPq)fS1=$W9T;EX<Hb_ddGB0P
z3dA{V|BePZK>1`k!vbuZO~k>6Wb&yn+MkE^*2FICg6}yhC<OSyB7WEp9<$t@I#T3r
zAzpAuwp%gfTdfPMYppUGkMny+jI)E)Lcl7GiGwfI!ZB+d%q(JUJ{(^T@Q{ZBLpX@+
ztHEA8%!5l=&?50kaB2PHZqrv49H8{>dDOHU4To_mb)`4h-o`X-Cuw(}R%w($1A(`+
zv0K`>+E3EZL{8h^i{oL5f6ezae`;)g(%?3!bQQ~F<PpNZ&0S%O74GqdaQS8116;;_
zX@43#H$5>1y$1b+uo?PYwL(rP_&GR%3=dZjk^v4;s!j2B9Fw%|Nu?UwU|lj8SPX>l
zDTZX^XhN?q#w(>VKOFS>Dw4!JF_2*#yhgV_qml!>5)df(5q?MpBDbbC!S@22Sh=<9
z?7Cnyip@$PwuQ7`8IVjp^i_}e#LoTc0?nQ{22QnUeGVsgQYoBiqj1sd6&|2fxx@CK
zJZ7k|aPgP{KL~=}AfgOYqTy-W$p<P6x@yo5t~{SySzli_!ETaXQ)nkc0*R4wl5yNz
zQe<QGQwZ$>1^p=3A|0Z4U*+0WQ7Q+0?WWYzGduzOWclF`iLQ%%^N((5uZu5ve2}+&
z(j+u#qx1;@uDv+7X{no{&}vo<w)Emq#kq<SB&l#1<vY6mCS%H^uPY|yWPs}|f3^*&
z&o(guFA^{_mOkn5Q$`!(@tINXVdHEWg7GD*!Pj`r@<9Y(E-^k^`u|6B=H4eNC$=-*
z{||6+();i!0-d}m1^CuF3k*mbKBn(3(SiCFIn0vI3-R6EiDTxwKo5EdJ-Jr~I?>3k
z56mnSqAT_+b%2pISuqI1dLRM&x9l7C1W1tfp7#%~{w=+{4y=^ObMyM{;_ebMDv=FV
z0rILd<efs_vO!*@;B=9awO7`6dQy>M;vM|6sDO0-HmMfX>FFaj@eel(pH72WE2w$W
z{!7_fmi1Rcu*t%Na`c($YOx^gBQdELb?8)uP8&rHNUb9$dD6SdCf;MV|GmxfvwJ>z
zw>U@+XQl{hzy?)m%SBqdw8gXUe~-~m@Pa;R%o4_U`5Nu@f*+Zym$Bt3*m4=R^rq*q
zn{y;Mdt;wBwZaBhERXL6-BD`pT>(&<bdy+?fwGh5!`m&~qSEwm9jQ5QZGj$J!$FpZ
zH#V-{3^#6t8y|<;y}{GC6^!5e`t6XLBX%BdbhRA&VJGVB#OuSo-li66Yy6WS7-bZw
z=DQ5A4{ms+0?#Mo+d<Hck{(pb2PUS=fPS<3ECSXsk)k4rddV2e9S1nb?-KS^lv>Bm
zDDPgorMW`Hd$<kav_$n3n@z4ANg+I53xZ*!j7qUm0~JJh@O*;xUf-ZKt_Nr#nLwCC
zb4)Ffii0jLQZh)=0O>cFtEk1Zt>*bRoT4b<YB9*PZGzj&hCSr)15itRkYu<G+XEyc
z?GBU99+47-9~f#U8pb8tRGvjDh3e`<I$D#&&q@4(1f>{t4FXeWVeLdpMTdrav<2GH
z{-dZjioc}8nGC0GIqDCajq>W&qWK#%-#0EX45aAnns?Q8lx>f+y$7$fm(x*ycAW=T
zkwLPy*8_2dDtZRZQ8US~?apZKBU+spqZD*8Y_Ep6qvt>Ruiz2-vbzvWEP;@0t45b8
z5EH!(;R{1n=(CZkJTCbAP6Iuj83h{SYq(hkVtlbAv3GB>+2jw+pZXa#YVxvJq<cO6
z=Msk64~bVtJ*wBjaDNo_${uwJGZZnW!heWNQLa@)h~}niE(_`+HQy)k0f}Eiv@eE6
zp?B-^mTioc%+2}uKL&Gi@Ni!96Ta*FUe$BF70>mT6pd<DN}PQ{yL|?s9e8MXSR!Se
azS6LF?V`Oq<vH(Cpk{QD^k7W$v;GgpuCi?a

literal 0
HcmV?d00001

diff --git a/scripts/model/autoencoder_backup.py b/scripts/model/autoencoder_backup.py
new file mode 100644
index 0000000..9e245ef
--- /dev/null
+++ b/scripts/model/autoencoder_backup.py
@@ -0,0 +1,38 @@
+import torch
+from torch.autograd import Variable
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+
+
+class DAE_Network(nn.Module):
+    def __init__(self, motion_dim, latent_dim):
+        super(DAE_Network, self).__init__()
+        print("init");
+        self.encoder = nn.Sequential(
+            nn.Linear(motion_dim, latent_dim),
+            nn.Tanh(),
+            # nn.Linear(motion_dim-10, motion_dim-30),
+            # nn.Tanh(),
+        )
+
+        self.decoder = nn.Sequential(
+            # nn.Linear(motion_dim-30, motion_dim-10),
+            # nn.Tanh(),
+            nn.Linear(latent_dim, motion_dim),
+
+        )
+
+    def forward(self, x):
+        # print("_________________")
+        # print(self.encoder)
+        # print(x.shape)
+        # print("_________________")
+        x = torch.squeeze(x)
+        # print(x.shape)
+        x = self.encoder(x)
+        # print("Encoded", x.shape)
+        x = self.decoder(x)
+        x = torch.unsqueeze(x, 2)
+        # print("Decoder", x.shape)
+        return x
\ No newline at end of file
diff --git a/scripts/model/seq2seq_net.py b/scripts/model/seq2seq_net.py
new file mode 100644
index 0000000..593a451
--- /dev/null
+++ b/scripts/model/seq2seq_net.py
@@ -0,0 +1,257 @@
+import torch
+from torch.autograd import Variable
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+
+'''
+Based on the following Se2Seq implementations:
+- https://github.com/AuCson/PyTorch-Batch-Attention-Seq2seq
+- https://github.com/spro/practical-pytorch/blob/master/seq2seq-translation/seq2seq-translation-batched.ipynb
+'''
+
+
+class EncoderRNN(nn.Module):
+    def __init__(self, input_size, embed_size, hidden_size, n_layers=1, dropout=0.5, pre_trained_embedding=None):
+        super(EncoderRNN, self).__init__()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.embed_size = embed_size
+        self.n_layers = n_layers
+        self.dropout = dropout
+
+        if pre_trained_embedding is not None:  # use pre-trained embedding (e.g., word2vec, glove)
+            assert pre_trained_embedding.shape[0] == input_size
+            assert pre_trained_embedding.shape[1] == embed_size
+            self.embedding = nn.Embedding.from_pretrained(torch.FloatTensor(pre_trained_embedding), freeze=False)
+        else:
+            self.embedding = nn.Embedding(input_size, embed_size)
+
+        self.gru = nn.GRU(embed_size, hidden_size, n_layers, dropout=self.dropout, bidirectional=True)
+
+        self.do_flatten_parameters = False
+        if torch.cuda.device_count() > 1:
+            self.do_flatten_parameters = True
+
+    def forward(self, input_seqs, input_lengths, hidden=None):
+        '''
+        :param input_seqs:
+            Variable of shape (num_step(T),batch_size(B)), sorted decreasingly by lengths(for packing)
+        :param input_lengths:
+            list of sequence length
+        :param hidden:
+            initial state of GRU
+        :returns:
+            GRU outputs in shape (T,B,hidden_size(H))
+            last hidden stat of RNN(i.e. last output for GRU)
+        '''
+        if self.do_flatten_parameters:
+            self.gru.flatten_parameters()
+
+        embedded = self.embedding(input_seqs)
+        packed = torch.nn.utils.rnn.pack_padded_sequence(embedded, input_lengths)
+        outputs, hidden = self.gru(packed, hidden)
+        outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(outputs)  # unpack (back to padded)
+        outputs = outputs[:, :, :self.hidden_size] + outputs[:, :, self.hidden_size:]  # Sum bidirectional outputs
+        return outputs, hidden
+
+
+class Attn(nn.Module):
+    def __init__(self, hidden_size):
+        super(Attn, self).__init__()
+        self.hidden_size = hidden_size
+        self.attn = nn.Linear(self.hidden_size * 2, hidden_size)
+        self.v = nn.Parameter(torch.rand(hidden_size))
+        stdv = 1. / math.sqrt(self.v.size(0))
+        self.v.data.normal_(mean=0, std=stdv)
+
+    def forward(self, hidden, encoder_outputs):
+        '''
+        :param hidden:
+            previous hidden state of the decoder, in shape (layers*directions,B,H)
+        :param encoder_outputs:
+            encoder outputs from Encoder, in shape (T,B,H)
+        :return
+            attention energies in shape (B,T)
+        '''
+        max_len = encoder_outputs.size(0)
+        this_batch_size = encoder_outputs.size(1)
+        H = hidden.repeat(max_len, 1, 1).transpose(0, 1)
+        encoder_outputs = encoder_outputs.transpose(0, 1)  # [B*T*H]
+        attn_energies = self.score(H, encoder_outputs)  # compute attention score
+        return F.softmax(attn_energies, dim=1).unsqueeze(1)  # normalize with softmax
+
+    def score(self, hidden, encoder_outputs):
+        energy = torch.tanh(self.attn(torch.cat([hidden, encoder_outputs], 2)))  # [B*T*2H]->[B*T*H]
+        energy = energy.transpose(2, 1)  # [B*H*T]
+        v = self.v.repeat(encoder_outputs.data.shape[0], 1).unsqueeze(1)  # [B*1*H]
+        energy = torch.bmm(v, energy)  # [B*1*T]
+        return energy.squeeze(1)  # [B*T]
+
+
+class BahdanauAttnDecoderRNN(nn.Module):
+    def __init__(self, input_size, hidden_size, output_size, n_layers=1, dropout_p=0.1,
+                 discrete_representation=False, speaker_model=None):
+        super(BahdanauAttnDecoderRNN, self).__init__()
+
+        # define parameters
+        self.hidden_size = hidden_size
+        self.output_size = output_size
+        self.n_layers = n_layers
+        self.dropout_p = dropout_p
+        self.discrete_representation = discrete_representation
+        self.speaker_model = speaker_model
+
+        # define embedding layer
+        if self.discrete_representation:
+            self.embedding = nn.Embedding(output_size, hidden_size)
+            self.dropout = nn.Dropout(dropout_p)
+
+        if self.speaker_model:
+            self.speaker_embedding = nn.Embedding(speaker_model.n_words, 8)
+
+        # calc input size
+        if self.discrete_representation:
+            input_size = hidden_size  # embedding size
+        linear_input_size = input_size + hidden_size
+        if self.speaker_model:
+            linear_input_size += 8
+
+        # define layers
+        self.pre_linear = nn.Sequential(
+            nn.Linear(linear_input_size, hidden_size),
+            nn.BatchNorm1d(hidden_size),
+            nn.ReLU(inplace=True)
+        )
+        self.attn = Attn(hidden_size)
+        self.gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=dropout_p)
+
+        # self.out = nn.Linear(hidden_size * 2, output_size)
+        self.out = nn.Linear(hidden_size, output_size)
+
+        self.do_flatten_parameters = False
+        if torch.cuda.device_count() > 1:
+            self.do_flatten_parameters = True
+
+    def freeze_attn(self):
+        for param in self.attn.parameters():
+            param.requires_grad = False
+
+    def forward(self, motion_input, last_hidden, encoder_outputs, vid_indices=None):
+        '''
+        :param motion_input:
+            motion input for current time step, in shape [batch x dim]
+        :param last_hidden:
+            last hidden state of the decoder, in shape [layers x batch x hidden_size]
+        :param encoder_outputs:
+            encoder outputs in shape [steps x batch x hidden_size]
+        :param vid_indices:
+        :return
+            decoder output
+        Note: we run this one step at a time i.e. you should use a outer loop
+            to process the whole sequence
+        '''
+
+        if self.do_flatten_parameters:
+            self.gru.flatten_parameters()
+
+        if self.discrete_representation:
+            word_embedded = self.embedding(motion_input).view(1, motion_input.size(0), -1)  # [1 x B x embedding_dim]
+            motion_input = self.dropout(word_embedded)
+        else:
+            motion_input = motion_input.view(1, motion_input.size(0), -1)  # [1 x batch x dim]
+
+        # attention
+        attn_weights = self.attn(last_hidden[-1], encoder_outputs)  # [batch x 1 x T]
+        context = attn_weights.bmm(encoder_outputs.transpose(0, 1))  # [batch x 1 x attn_size]
+        context = context.transpose(0, 1)  # [1 x batch x attn_size]
+
+        # make input vec
+        rnn_input = torch.cat((motion_input, context), 2)  # [1 x batch x (dim + attn_size)]
+
+        if self.speaker_model:
+            assert vid_indices is not None
+            speaker_context = self.speaker_embedding(vid_indices).unsqueeze(0)
+            rnn_input = torch.cat((rnn_input, speaker_context), 2)  # [1 x batch x (dim + attn_size + embed_size)]
+
+        rnn_input = self.pre_linear(rnn_input.squeeze(0))
+        rnn_input = rnn_input.unsqueeze(0)
+
+        # rnn
+        output, hidden = self.gru(rnn_input, last_hidden)
+        # print("rnn_input", rnn_input.shape)
+        # print("last_hidden", last_hidden.shape)
+        # print(output.shape)
+        # post-fc
+
+        output = output.squeeze(0)  # [1 x batch x hidden_size] -> [batch x hidden_size]
+        output = self.out(output)
+
+        return output, hidden, attn_weights
+
+
+class Generator(nn.Module):
+    def __init__(self, args, motion_dim, discrete_representation=False, speaker_model=None):
+        super(Generator, self).__init__()
+        self.output_size = motion_dim
+        self.n_layers = args.n_layers
+        self.discrete_representation = discrete_representation
+        self.decoder = BahdanauAttnDecoderRNN(input_size=motion_dim,
+                                              hidden_size=args.hidden_size,
+                                              output_size=self.output_size,
+                                              n_layers=self.n_layers,
+                                              dropout_p=args.dropout_prob,
+                                              discrete_representation=discrete_representation,
+                                              speaker_model=speaker_model)
+
+    def freeze_attn(self):
+        self.decoder.freeze_attn()
+
+    def forward(self, z, motion_input, last_hidden, encoder_output, vid_indices=None):
+        if z is None:
+            input_with_noise_vec = motion_input
+        else:
+            assert not self.discrete_representation  # not valid for discrete representation
+            input_with_noise_vec = torch.cat([motion_input, z], dim=1)  # [bs x (10+z_size)]
+
+        return self.decoder(input_with_noise_vec, last_hidden, encoder_output, vid_indices)
+
+
+class Seq2SeqNet(nn.Module):
+    def __init__(self, args, pose_dim, n_frames, n_words, word_embed_size, word_embeddings, speaker_model=None):
+        super().__init__()
+        self.encoder = EncoderRNN(
+            n_words, word_embed_size, args.hidden_size, args.n_layers,
+            dropout=args.dropout_prob, pre_trained_embedding=word_embeddings)
+        self.decoder = Generator(args, pose_dim, speaker_model=speaker_model)
+
+        self.n_frames = n_frames
+        self.n_pre_poses = args.n_pre_poses
+        self.pose_dim = pose_dim
+
+    def forward(self, in_text, in_lengths, poses, vid_indices):
+        # reshape to (seq x batch x dim)
+        in_text = in_text.transpose(0, 1)
+        poses = poses.transpose(0, 1)
+
+        outputs = torch.zeros(self.n_frames, poses.size(1), self.decoder.output_size).to(poses.device)
+
+        # run words through encoder
+        encoder_outputs, encoder_hidden = self.encoder(in_text, in_lengths, None)
+        decoder_hidden = encoder_hidden[:self.decoder.n_layers]  # use last hidden state from encoder
+
+        # run through decoder one time step at a time
+        decoder_input = poses[0]  # initial pose from the dataset
+        outputs[0] = decoder_input
+
+        for t in range(1, self.n_frames):
+            decoder_output, decoder_hidden, _ = self.decoder(None, decoder_input, decoder_hidden, encoder_outputs,
+                                                             vid_indices)
+            outputs[t] = decoder_output
+
+            if t < self.n_pre_poses:
+                decoder_input = poses[t]  # next input is current target
+            else:
+                decoder_input = decoder_output  # next input is current prediction
+
+        return outputs.transpose(0, 1)
diff --git a/scripts/model/seq2seq_with_cluster_model.py b/scripts/model/seq2seq_with_cluster_model.py
new file mode 100644
index 0000000..a0b8de6
--- /dev/null
+++ b/scripts/model/seq2seq_with_cluster_model.py
@@ -0,0 +1,71 @@
+import torch
+from torch.autograd import Variable
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+
+
+class cluster2gesture_model(nn.Module):
+    def __init__(self, args, input_size, embed_size, hidden_size, output_size, n_layers=1, dropout=0.5, pre_trained_embedding=None):
+        super(cluster2gesture_model, self).__init__()
+        self.input_size = input_size
+        self.output_size = output_size
+        self.hidden_size = hidden_size
+        self.embed_size = embed_size
+        self.n_layers = n_layers
+        self.dropout = dropout
+        self.n_frames = args.n_poses
+
+        self.embedding = nn.Embedding(input_size, embed_size)
+        assert(hidden_size==embed_size)
+
+        self.pre_gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=self.dropout,
+                          batch_first=True)
+
+
+        self.pre_linear = nn.Sequential(
+            nn.Linear(output_size, hidden_size),
+            nn.BatchNorm1d(hidden_size),
+            nn.ReLU(inplace=True)
+        )
+        self.gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=self.dropout,
+                          batch_first=True)
+        self.out_layer = nn.Linear(hidden_size, output_size)
+
+    def forward(self, input_cluster, out_poses):
+        '''
+        :param input_seqs:
+            Variable of shape (num_step(T),batch_size(B)), sorted decreasingly by lengths(for packing)
+        :param input_lengths:
+            list of sequence length
+        :param hidden:
+            initial state of GRU
+        :returns:
+            GRU outputs in shape (T,B,hidden_size(H))
+            last hidden stat of RNN(i.e. last output for GRU)
+        '''
+        outputs = torch.zeros_like(out_poses).to(out_poses.device)
+
+
+        embedded = self.embedding(input_cluster)
+        # packed = torch.nn.utils.rnn.pack_padded_sequence(embedded, input_lengths)
+        embedded = embedded.unsqueeze(1)
+        encoder_output, encoder_hidden = self.pre_gru(embedded, None)
+
+        decoder_input = torch.zeros_like(out_poses[:,0,:])
+        decode_hidden = encoder_hidden
+        for t in range(1, self.n_frames):
+            decoder_input = self.pre_linear(decoder_input)
+            decoder_input = decoder_input.unsqueeze(1) #since batch_first=true, B*82 -> b*L*82
+
+            decoder_output, decoder_hidden = self.gru(decoder_input, decode_hidden)
+            # post-fc
+            decoder_output = decoder_output.squeeze(1)  # [1 x batch x hidden_size] -> [batch x hidden_size]
+            output = self.out_layer(decoder_output)
+
+            outputs[:, t, :] = output
+
+            decoder_input = output  # next input is current prediction
+
+        return outputs
+
diff --git a/scripts/model/tcn.py b/scripts/model/tcn.py
new file mode 100644
index 0000000..afd8435
--- /dev/null
+++ b/scripts/model/tcn.py
@@ -0,0 +1,64 @@
+""" from https://github.com/locuslab/TCN/blob/master/TCN/tcn.py """
+import torch
+import torch.nn as nn
+from torch.nn.utils import weight_norm
+
+
+class Chomp1d(nn.Module):
+    def __init__(self, chomp_size):
+        super(Chomp1d, self).__init__()
+        self.chomp_size = chomp_size
+
+    def forward(self, x):
+        return x[:, :, :-self.chomp_size].contiguous()
+
+
+class TemporalBlock(nn.Module):
+    def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation, padding, dropout=0.2):
+        super(TemporalBlock, self).__init__()
+        self.conv1 = weight_norm(nn.Conv1d(n_inputs, n_outputs, kernel_size,
+                                           stride=stride, padding=padding, dilation=dilation))
+        self.chomp1 = Chomp1d(padding)
+        self.relu1 = nn.ReLU()
+        self.dropout1 = nn.Dropout(dropout)
+
+        self.conv2 = weight_norm(nn.Conv1d(n_outputs, n_outputs, kernel_size,
+                                           stride=stride, padding=padding, dilation=dilation))
+        self.chomp2 = Chomp1d(padding)
+        self.relu2 = nn.ReLU()
+        self.dropout2 = nn.Dropout(dropout)
+
+        self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1, self.dropout1,
+                                 self.conv2, self.chomp2, self.relu2, self.dropout2)
+        self.downsample = nn.Conv1d(n_inputs, n_outputs, 1) if n_inputs != n_outputs else None
+        self.relu = nn.ReLU()
+        self.init_weights()
+
+    def init_weights(self):
+        self.conv1.weight.data.normal_(0, 0.01)
+        self.conv2.weight.data.normal_(0, 0.01)
+        if self.downsample is not None:
+            self.downsample.weight.data.normal_(0, 0.01)
+
+    def forward(self, x):
+        out = self.net(x)
+        res = x if self.downsample is None else self.downsample(x)
+        return self.relu(out + res)
+
+
+class TemporalConvNet(nn.Module):
+    def __init__(self, num_inputs, num_channels, kernel_size=2, dropout=0.2):
+        super(TemporalConvNet, self).__init__()
+        layers = []
+        num_levels = len(num_channels)
+        for i in range(num_levels):
+            dilation_size = 2 ** i
+            in_channels = num_inputs if i == 0 else num_channels[i-1]
+            out_channels = num_channels[i]
+            layers += [TemporalBlock(in_channels, out_channels, kernel_size, stride=1, dilation=dilation_size,
+                                     padding=(kernel_size-1) * dilation_size, dropout=dropout)]
+
+        self.network = nn.Sequential(*layers)
+
+    def forward(self, x):
+        return self.network(x)
\ No newline at end of file
diff --git a/scripts/model/text2embedding_GAN_model.py b/scripts/model/text2embedding_GAN_model.py
new file mode 100644
index 0000000..f67feb1
--- /dev/null
+++ b/scripts/model/text2embedding_GAN_model.py
@@ -0,0 +1,636 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+import numpy as np
+import torch
+from torch.autograd import Variable
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+
+'''
+Based on the following Se2Seq implementations:
+- https://github.com/AuCson/PyTorch-Batch-Attention-Seq2seq
+- https://github.com/spro/practical-pytorch/blob/master/seq2seq-translation/seq2seq-translation-batched.ipynb
+'''
+debug = False
+
+class EncoderRNN(nn.Module):
+    def __init__(self, input_size, embed_size, hidden_size, n_layers=1, dropout=0.5, pre_trained_embedding=None):
+        super(EncoderRNN, self).__init__()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.embed_size = embed_size
+        self.n_layers = n_layers
+        self.dropout = dropout
+
+        if pre_trained_embedding is not None:  # use pre-trained embedding (e.g., word2vec, glove)
+            assert pre_trained_embedding.shape[0] == input_size
+            assert pre_trained_embedding.shape[1] == embed_size
+            self.embedding = nn.Embedding.from_pretrained(torch.FloatTensor(pre_trained_embedding), freeze=False)
+        else:
+            self.embedding = nn.Embedding(input_size, embed_size)
+
+        self.gru = nn.GRU(embed_size, hidden_size, n_layers, dropout=self.dropout, bidirectional=True)
+
+        self.do_flatten_parameters = False
+        if torch.cuda.device_count() > 1:
+            self.do_flatten_parameters = True
+
+    def forward(self, input_seqs, input_lengths, hidden=None):
+        '''
+        :param input_seqs:
+            Variable of shape (num_step(T),batch_size(B)), sorted decreasingly by lengths(for packing)
+        :param input_lengths:
+            list of sequence length
+        :param hidden:
+            initial state of GRU
+        :returns:
+            GRU outputs in shape (T,B,hidden_size(H))
+            last hidden stat of RNN(i.e. last output for GRU)
+        '''
+        if self.do_flatten_parameters:
+            self.gru.flatten_parameters()
+
+        embedded = self.embedding(input_seqs)
+        packed = torch.nn.utils.rnn.pack_padded_sequence(embedded, input_lengths)
+        outputs, hidden = self.gru(packed, hidden)
+        outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(outputs)  # unpack (back to padded)
+        outputs = outputs[:, :, :self.hidden_size] + outputs[:, :, self.hidden_size:]  # Sum bidirectional outputs
+        return outputs, hidden
+
+
+class Attn(nn.Module):
+    def __init__(self, hidden_size):
+        super(Attn, self).__init__()
+        self.hidden_size = hidden_size
+        self.attn = nn.Linear(self.hidden_size * 2, hidden_size)
+        self.v = nn.Parameter(torch.rand(hidden_size))
+        stdv = 1. / math.sqrt(self.v.size(0))
+        self.v.data.normal_(mean=0, std=stdv)
+
+    def forward(self, hidden, encoder_outputs):
+        '''
+        :param hidden:
+            previous hidden state of the decoder, in shape (layers*directions,B,H)
+        :param encoder_outputs:
+            encoder outputs from Encoder, in shape (T,B,H)
+        :return
+            attention energies in shape (B,T)
+        '''
+        max_len = encoder_outputs.size(0)
+        this_batch_size = encoder_outputs.size(1)
+        H = hidden.repeat(max_len, 1, 1).transpose(0, 1)
+        encoder_outputs = encoder_outputs.transpose(0, 1)  # [B*T*H]
+        attn_energies = self.score(H, encoder_outputs)  # compute attention score
+        return F.softmax(attn_energies, dim=1).unsqueeze(1)  # normalize with softmax
+
+    def score(self, hidden, encoder_outputs):
+        energy = torch.tanh(self.attn(torch.cat([hidden, encoder_outputs], 2)))  # [B*T*2H]->[B*T*H]
+        energy = energy.transpose(2, 1)  # [B*H*T]
+        v = self.v.repeat(encoder_outputs.data.shape[0], 1).unsqueeze(1)  # [B*1*H]
+        energy = torch.bmm(v, energy)  # [B*1*T]
+        return energy.squeeze(1)  # [B*T]
+
+
+class BahdanauAttnDecoderRNN(nn.Module):
+    def __init__(self, input_size, hidden_size, output_size, n_layers=1, dropout_p=0.1,
+                 discrete_representation=False, speaker_model=None):
+        super(BahdanauAttnDecoderRNN, self).__init__()
+
+        # define parameters
+        self.hidden_size = hidden_size
+        if noisy:
+            self.hidden_size = hidden_size*2
+        self.output_size = output_size
+        self.n_layers = n_layers
+        self.dropout_p = dropout_p
+        self.discrete_representation = discrete_representation
+        self.speaker_model = speaker_model
+
+        # define embedding layer
+        if self.discrete_representation:
+            self.embedding = nn.Embedding(output_size, hidden_size)
+            self.dropout = nn.Dropout(dropout_p)
+
+        if self.speaker_model:
+            self.speaker_embedding = nn.Embedding(speaker_model.n_words, 8)
+
+        # calc input size
+        if self.discrete_representation:
+            input_size = hidden_size  # embedding size
+        linear_input_size = input_size + hidden_size
+        if self.speaker_model:
+            linear_input_size += 8
+
+        # define layers
+        self.pre_linear = nn.Sequential(
+            nn.Linear(linear_input_size, hidden_size),
+            nn.BatchNorm1d(hidden_size),
+            nn.ReLU(inplace=True)
+        )
+        self.attn = Attn(hidden_size)
+        self.gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=dropout_p)
+
+        # self.out = nn.Linear(hidden_size * 2, output_size)
+        self.out = nn.Linear(hidden_size, output_size)
+
+        self.do_flatten_parameters = False
+        if torch.cuda.device_count() > 1:
+            self.do_flatten_parameters = True
+
+    def freeze_attn(self):
+        for param in self.attn.parameters():
+            param.requires_grad = False
+
+    def forward(self, motion_input, last_hidden, encoder_outputs, vid_indices=None):
+        '''
+        :param motion_input:
+            motion input for current time step, in shape [batch x dim]
+        :param last_hidden:
+            last hidden state of the decoder, in shape [layers x batch x hidden_size]
+        :param encoder_outputs:
+            encoder outputs in shape [steps x batch x hidden_size]
+        :param vid_indices:
+        :return
+            decoder output
+        Note: we run this one step at a time i.e. you should use a outer loop
+            to process the whole sequence
+        '''
+
+        if self.do_flatten_parameters:
+            self.gru.flatten_parameters()
+
+        if self.discrete_representation:
+            word_embedded = self.embedding(motion_input).view(1, motion_input.size(0), -1)  # [1 x B x embedding_dim]
+            motion_input = self.dropout(word_embedded)
+        else:
+            motion_input = motion_input.view(1, motion_input.size(0), -1)  # [1 x batch x dim]
+
+        # attention
+        attn_weights = self.attn(last_hidden[-1], encoder_outputs)  # [batch x 1 x T]
+        context = attn_weights.bmm(encoder_outputs.transpose(0, 1))  # [batch x 1 x attn_size]
+        context = context.transpose(0, 1)  # [1 x batch x attn_size]
+
+        # make input vec
+        rnn_input = torch.cat((motion_input, context), 2)  # [1 x batch x (dim + attn_size)]
+
+        if self.speaker_model:
+            assert vid_indices is not None
+            speaker_context = self.speaker_embedding(vid_indices).unsqueeze(0)
+            rnn_input = torch.cat((rnn_input, speaker_context), 2)  # [1 x batch x (dim + attn_size + embed_size)]
+
+        rnn_input = self.pre_linear(rnn_input.squeeze(0))
+        rnn_input = rnn_input.unsqueeze(0)
+
+        # rnn
+        output, hidden = self.gru(rnn_input, last_hidden)
+        if debug:
+            print("rnn_input", rnn_input.shape)
+            print("last_hidden", last_hidden.shape)
+            print(output.shape)
+        # post-fc
+
+        output = output.squeeze(0)  # [1 x batch x hidden_size] -> [batch x hidden_size]
+        output = self.out((output))
+
+        return output, hidden, attn_weights
+
+
+class Generator(nn.Module):
+    def __init__(self, args, motion_dim, discrete_representation=False, speaker_model=None):
+        super(Generator, self).__init__()
+        self.output_size = motion_dim
+        self.n_layers = args.n_layers
+        self.discrete_representation = discrete_representation
+        self.decoder = BahdanauAttnDecoderRNN(input_size=motion_dim,
+                                              hidden_size=args.hidden_size,
+                                              output_size=self.output_size,
+                                              n_layers=self.n_layers,
+                                              dropout_p=args.dropout_prob,
+                                              discrete_representation=discrete_representation,
+                                              speaker_model=speaker_model)
+
+    def freeze_attn(self):
+        self.decoder.freeze_attn()
+
+    def forward(self, z, motion_input, last_hidden, encoder_output, vid_indices=None):
+        if z is None:
+            input_with_noise_vec = motion_input
+        else:
+            assert not self.discrete_representation  # not valid for discrete representation
+            input_with_noise_vec = torch.cat([motion_input, z], dim=1)  # [bs x (10+z_size)]
+
+        return self.decoder(input_with_noise_vec, last_hidden, encoder_output, vid_indices)
+
+noisy = False
+class text2embedding_model(nn.Module):
+    def __init__(self, args, pose_dim, n_frames, n_words, word_embed_size, word_embeddings, speaker_model=None):
+        super().__init__()
+
+        # Todo: find a better way to play with representation learning dimension
+        # Todo: Regarding hidden size, sentence time length, etc.
+        pose_dim = args.n_layers * args.hidden_size
+
+        self.encoder = EncoderRNN(
+            n_words, word_embed_size, args.hidden_size, args.n_layers,
+            dropout=args.dropout_prob, pre_trained_embedding=word_embeddings)
+        self.decoder = Generator(args, pose_dim, speaker_model=speaker_model)
+
+        self.n_frames = n_frames
+        self.n_pre_poses = args.n_pre_poses
+        self.pose_dim = pose_dim
+        self.sentence_frame_length = args.sentence_frame_length
+
+    def forward(self, in_text, in_lengths, poses, vid_indices):
+        # reshape to (seq x batch x dim)
+        in_text = in_text.transpose(0, 1)
+        poses = poses.transpose(0, 1)
+
+
+        outputs = torch.zeros(self.sentence_frame_length//self.n_frames, poses.size(1),
+                              self.decoder.output_size)\
+            .to(poses.device)
+        if debug:
+            print("outputs", outputs.shape)
+
+
+
+        # run words through encoder
+        encoder_outputs, encoder_hidden = self.encoder(in_text, in_lengths, None)
+        decoder_hidden = encoder_hidden[:self.decoder.n_layers]  # use last hidden state from encoder
+        if noisy:
+            eps = torch.randn_like(decoder_hidden)
+            decoder_hidden = eps*10 # torch.cat([decoder_hidden, eps/10], dim=2)
+
+        # run through decoder one time step at a time
+        # decoder_input = poses[0]  # initial pose from the dataset
+        decoder_input = torch.zeros(( poses.size(1), self.decoder.output_size) ).to(poses.device)
+        outputs[0] = decoder_input
+
+        q = self.sentence_frame_length//self.n_frames
+        if debug:
+            print("self.sentence_frame_length//self.n_frames", self.sentence_frame_length//self.n_frames)
+        for t in range(0, self.sentence_frame_length//self.n_frames):
+            decoder_output, decoder_hidden, _ = self.decoder(None, decoder_input, decoder_hidden, encoder_outputs,
+                                                             vid_indices)
+            outputs[t] = decoder_output
+
+            # if t < self.n_pre_poses:
+            #     decoder_input = poses[t]  # next input is current target
+            # else:
+            decoder_input = decoder_output  # next input is current prediction
+
+        return outputs.transpose(0, 1)
+
+
+class Generator_gan(nn.Module):
+
+    def __init__(self, i_size, n_size, o_size, h_size):
+        super(Generator, self).__init__()
+
+        self.i_size = i_size
+        self.n_size = n_size
+        self.o_size = o_size
+
+        self.i_fc = nn.Linear(i_size, int(h_size / 2))
+        self.n_fc = nn.Linear(n_size, int(h_size / 2))
+        self.o_fc = nn.Linear(2 * h_size, o_size)
+        self.rnn = nn.LSTM(h_size, h_size, num_layers=2, bidirectional=True)
+
+    def forward(self, i, n):
+        """3D tensor"""
+
+        assert len(
+            i.shape) == 3, f"expect 3D tensor with shape (t, n, dim), got {i.shape}"
+        assert n.size(
+            0) == 1, f"shape of noise must be (1, N, dim), got noise with {n.size(0)}"
+        assert i.size(1) == n.size(
+            1), f"batch size of input and noise must be the same, got input with {i.size(1)}, noise with {n.size(1)}"
+
+        n = n.repeat(i.size(0), 1, 1)
+        t, bs = i.size(0), i.size(1)
+        i = i.view(t * bs, -1)
+        n = n.view(t * bs, -1)
+        i = F.leaky_relu(self.i_fc(i), 1e-2)
+        n = F.leaky_relu(self.n_fc(n), 1e-2)
+        i = i.view(t, bs, -1)
+        n = n.view(t, bs, -1)
+        x = torch.cat([i, n], dim=-1)
+        x, _ = self.rnn(x)
+        x = F.leaky_relu(x, 1e-2)
+        o = self.o_fc(x)
+        return o
+
+    def forward_given_noise_seq(self, i, n):
+        t, bs = i.size(0), i.size(1)
+        i = i.view(t * bs, -1)
+        n = n.view(t * bs, -1)
+        i = F.leaky_relu(self.i_fc(i), 1e-2)
+        n = F.leaky_relu(self.n_fc(n), 1e-2)
+        i = i.view(t, bs, -1)
+        n = n.view(t, bs, -1)
+        x = torch.cat([i, n], dim=-1)
+        x, _ = self.rnn(x)
+        x = F.leaky_relu(x, 1e-2)
+        o = self.o_fc(x)
+        return o
+
+    def count_parameters(self):
+        return sum([p.numel() for p in self.parameters() if p.requires_grad])
+
+
+class Discriminator_gan(nn.Module):
+
+    def __init__(self, i_size, c_size, h_size):
+        super(Discriminator_gan, self).__init__()
+
+        self.i_size = i_size
+        self.c_szie = c_size
+
+        self.i_fc = nn.Linear(i_size, int(h_size / 2))
+        self.c_fc = nn.Linear(c_size, int(h_size / 2))
+        self.rnn = nn.LSTM(h_size, h_size, num_layers=2, bidirectional=True)
+        self.o_fc = nn.Linear(2 * h_size, 1)
+
+    def forward(self, x, c):
+        """3D tensor + 3D tensor"""
+
+        assert len(x.shape) == 3, f"expect 3D tensor, got {x.shape}"
+
+        t, bs = x.size(0), x.size(1)
+        x = x.view(t * bs, -1)
+        c = c.view(t * bs, -1)
+        x = F.leaky_relu(self.i_fc(x))
+        c = F.leaky_relu(self.c_fc(c))
+        x = x.view(t, bs, -1)
+        c = c.view(t, bs, -1)
+        x = torch.cat([x, c], dim=-1)
+        x, _ = self.rnn(x)
+        x = x.view(t * bs, -1)
+        x = torch.sigmoid(self.o_fc(x))
+        x = x.view(t, bs, -1)
+        return x
+
+    def count_parameters(self):
+        return sum([p.numel() for p in self.parameters() if p.requires_grad])
+
+
+class EncoderRNN_discriminator(nn.Module):
+    def __init__(self, input_size, embed_size, hidden_size, n_layers=1, dropout=0.5, pre_trained_embedding=None):
+        super(EncoderRNN, self).__init__()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.embed_size = embed_size
+        self.n_layers = n_layers
+        self.dropout = dropout
+
+        if pre_trained_embedding is not None:  # use pre-trained embedding (e.g., word2vec, glove)
+            assert pre_trained_embedding.shape[0] == input_size
+            assert pre_trained_embedding.shape[1] == embed_size
+            self.embedding = nn.Embedding.from_pretrained(torch.FloatTensor(pre_trained_embedding), freeze=False)
+        else:
+            self.embedding = nn.Embedding(input_size, embed_size)
+
+        self.gru = nn.GRU(embed_size, hidden_size, n_layers, dropout=self.dropout, bidirectional=True)
+
+        self.f_discriminate = nn.Linear(n_layers * hidden_size, 1)
+
+        self.do_flatten_parameters = False
+        if torch.cuda.device_count() > 1:
+            self.do_flatten_parameters = True
+
+    def forward(self, input_seqs, input_lengths, hidden=None):
+        '''
+        :param input_seqs:
+            Variable of shape (num_step(T),batch_size(B)), sorted decreasingly by lengths(for packing)
+        :param input_lengths:
+            list of sequence length
+        :param hidden:
+            initial state of GRU
+        :returns:
+            GRU outputs in shape (T,B,hidden_size(H))
+            last hidden stat of RNN(i.e. last output for GRU)
+        '''
+        if self.do_flatten_parameters:
+            self.gru.flatten_parameters()
+
+        embedded = self.embedding(input_seqs)
+        packed = torch.nn.utils.rnn.pack_padded_sequence(embedded, input_lengths)
+        outputs, hidden = self.gru(packed, hidden)
+        outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(outputs)  # unpack (back to padded)
+        outputs = outputs[:, :, :self.hidden_size] + outputs[:, :, self.hidden_size:]  # Sum bidirectional outputs
+        return outputs, hidden
+
+
+class text2embedding_model_generator(nn.Module):
+    def __init__(self, args, pose_dim, n_frames, n_words, word_embed_size, word_embeddings, speaker_model=None):
+        super().__init__()
+
+        # Todo: find a better way to play with representation learning dimension
+        # Todo: Regarding hidden size, sentence time length, etc.
+
+        pose_dim = args.n_layers * args.hidden_size
+        pose_dim = 400
+        self.encoder_text = EncoderRNN(
+            n_words, word_embed_size, args.hidden_size // 2, args.n_layers,
+            dropout=args.dropout_prob, pre_trained_embedding=word_embeddings)
+        self.decoder = Generator(args, pose_dim, speaker_model=speaker_model)
+
+        self.fc_input = nn.Linear(args.hidden_size // 2, args.hidden_size // 2)
+        self.fc_noise = nn.Linear(args.noise_dim, args.hidden_size // 2)
+
+        self.n_frames = n_frames
+        self.n_pre_poses = args.n_pre_poses
+        self.pose_dim = pose_dim
+        self.sentence_frame_length = args.sentence_frame_length
+        self.noise_dim = args.noise_dim
+    def forward(self, in_text, in_lengths, poses, vid_indices, ):
+        # reshape to (seq x batch x dim)
+        in_text = in_text.transpose(0, 1)
+        poses = poses.transpose(0, 1)
+
+        outputs = torch.zeros(self.sentence_frame_length // self.n_frames, poses.size(1),
+                              self.decoder.output_size) \
+            .to(poses.device)
+        if debug:
+            print("outputs", outputs.shape)
+
+        # run words through encoder
+        encoder_outputs, encoder_hidden = self.encoder_text(in_text, in_lengths, None)
+        if debug:
+            print("encoder_hidden from lstm", encoder_hidden.shape)
+            print("Encoder_text", self.encoder_text)
+        encoder_hidden = F.leaky_relu(self.fc_input(encoder_hidden), 1e-2)
+
+        noise = np.random.normal(0, 1, (encoder_hidden.shape[0]* encoder_hidden.shape[1], self.noise_dim))
+        noise = torch.from_numpy(noise).float().to(encoder_hidden.device)
+        if debug:
+            print("Zarb", encoder_hidden.shape[0]* encoder_hidden.shape[1])
+            print("noise.shape1", noise.shape)
+            print("encoder_hidden.shape1", encoder_hidden.shape)
+        noise = F.leaky_relu(self.fc_noise(noise), 1e-2)
+        if debug:
+            print("after leaky", noise.shape)
+        noise = noise.reshape(encoder_hidden.shape)
+        if debug:
+            print("noise2", noise.shape)
+            print("encoder_hidden2", encoder_hidden.shape)
+
+        decoder_hidden = torch.cat([encoder_hidden[:self.decoder.n_layers],
+                                    noise[:self.decoder.n_layers]], dim=-1)
+
+        encoder_outputs = torch.cat([encoder_outputs[:self.decoder.n_layers],
+                                    noise[:self.decoder.n_layers]], dim=-1)
+        if debug:
+            print(" cat !decoder_hidden", decoder_hidden.shape)
+
+
+        # decoder_hidden = encoder_hidden[:self.decoder.n_layers]  # use last hidden state from encoder
+        if noisy:
+            eps = torch.randn_like(decoder_hidden)
+            decoder_hidden = eps * 10  # torch.cat([decoder_hidden, eps/10], dim=2)
+
+        # run through decoder one time step at a time
+        # decoder_input = poses[0]  # initial pose from the dataset
+        decoder_input = torch.zeros((poses.size(1), self.decoder.output_size)).to(poses.device)
+        outputs[0] = decoder_input
+
+        q = self.sentence_frame_length // self.n_frames
+        if debug:
+            print("self.sentence_frame_length//self.n_frames", self.sentence_frame_length // self.n_frames)
+        for t in range(0, self.sentence_frame_length // self.n_frames):
+            decoder_output, decoder_hidden, _ = self.decoder(None, decoder_input, decoder_hidden, encoder_outputs,
+                                                             vid_indices)
+            outputs[t] = decoder_output
+
+            # if t < self.n_pre_poses:
+            #     decoder_input = poses[t]  # next input is current target
+            # else:
+            decoder_input = decoder_output  # next input is current prediction
+
+        return outputs.transpose(0, 1)
+
+
+class text2embedding_model_discriminator(nn.Module):
+    def __init__(self, args, pose_dim, n_frames, n_words, word_embed_size, word_embeddings, speaker_model=None):
+        super().__init__()
+
+        # Todo: find a better way to play with representation learning dimension
+        # Todo: Regarding hidden size, sentence time length, etc.
+        pose_dim = args.n_layers * args.hidden_size
+
+        self.encoder_text = EncoderRNN(
+            n_words, word_embed_size, args.hidden_size // 2, args.n_layers,
+            dropout=args.dropout_prob, pre_trained_embedding=word_embeddings)
+        # self.decoder = Generator(args, pose_dim, speaker_model=speaker_model)
+
+        # Todo: fix this 400 dim
+        self.encoder_pose_latent = nn.GRU(400, args.hidden_size // 2, args.n_layers,
+                                          dropout=args.dropout_prob, bidirectional=True)
+
+        self.decoder = nn.GRU(args.hidden_size, args.hidden_size, args.n_layers,
+                              dropout=args.dropout_prob, bidirectional=True)
+
+        self.fc_text = nn.Linear(args.hidden_size // 2, args.hidden_size // 2)
+        self.fc_pose = nn.Linear(args.hidden_size // 2, args.hidden_size // 2)
+        self.classifier = nn.Linear(args.hidden_size * args.n_layers, 1)
+
+        self.n_frames = n_frames
+        self.n_pre_poses = args.n_pre_poses
+        self.pose_dim = pose_dim
+        self.sentence_frame_length = args.sentence_frame_length
+        self.n_layers = args.n_layers
+    def forward(self, in_text, in_lengths, poses, vid_indices):
+        # reshape to (seq x batch x dim)
+        in_text = in_text.transpose(0, 1)
+        poses = poses.transpose(0, 1)
+
+        # outputs = torch.zeros(self.sentence_frame_length // self.n_frames, poses.size(1),
+        #                       self.decoder.output_size) \
+        #     .to(poses.device)
+        # if debug:
+        #     print("outputs", outputs.shape)
+
+        # run words through encoder
+        # 1. Encoders
+        encoder_text_outputs, encoder_text_hidden = self.encoder_text(in_text, in_lengths, None)
+        encoder_pose_outputs, encoder_pose_hidden = self.encoder_pose_latent(poses, None)
+        encoder_text_hidden = torch.nn.functional.leaky_relu(self.fc_text(encoder_text_hidden), 1e-2)
+        encoder_pose_hidden = torch.nn.functional.leaky_relu(self.fc_pose(encoder_pose_hidden), 1e-2)
+
+        if debug:
+            print("encoder_text_outputs", encoder_text_outputs.shape)
+            print("encoder_pose_outputs", encoder_pose_outputs.shape)
+        # 2. Decoder
+        decoder_hidden = torch.cat([encoder_text_hidden[:self.n_layers],
+                                    encoder_pose_hidden[:self.n_layers]], dim=-1)
+
+
+        # decoder_hidden = encoder_hidden[:self.decoder.n_layers]  # use last hidden state from encoder
+        if noisy:
+            eps = torch.randn_like(decoder_hidden)
+            decoder_hidden = eps * 10  # torch.cat([decoder_hidden, eps/10], dim=2)
+
+
+        # Todo: should find a way to make surw that if it's ok to use pose encoder output, not text or combined
+        # decoder_input = torch.zeros_like(poses)
+        decoder_input = encoder_pose_outputs
+        if self.decoder.bidirectional:
+            decoder_hidden = torch.cat([decoder_hidden, decoder_hidden], dim=0)
+        if debug:
+            print("D: decoder_input", decoder_input.shape)
+            print("D decoder_hidden", decoder_hidden.shape)
+            print(self.decoder)
+
+        decoder_outputs, decoder_hidden = self.decoder(decoder_input, decoder_hidden)
+
+        # run through decoder one time step at a time
+        # decoder_input = poses[0]  # initial pose from the dataset
+        # decoder_input = torch.zeros(( poses.size(1), self.decoder.output_size) ).to(poses.device)
+        # outputs[0] = decoder_input
+
+        if debug:
+            print("Final decoder hidden", decoder_hidden.shape)
+            print("Final calssifier", self.classifier)
+        decoder_hidden = decoder_hidden[:self.n_layers]
+        decoder_hidden = decoder_hidden.transpose(1,0)
+        decoder_hidden = decoder_hidden.reshape(decoder_hidden.shape[0], -1)
+        if debug:
+            print("Final decoder hidden", decoder_hidden.shape)
+
+        test = (self.classifier(decoder_hidden))
+        test = test.squeeze()
+        if debug:
+            print("test squeeze", test.shape)
+        test =torch.sigmoid(test)
+        if debug:
+            print("test sigmoid", test.shape)
+        output = torch.sigmoid(self.classifier(decoder_hidden.squeeze()))
+        if debug:
+            print("TEsT:,", test.shape)
+            print("Discriminator output", output.shape)
+        return output
+        # return outputs.transpose(0, 1)
+
+
+
+
+
+
+
+
+
+class text2embedding_model_gan(nn.Module):
+    def __init__(self, args, pose_dim, n_frames, n_words, word_embed_size, word_embeddings, speaker_model=None):
+        super().__init__()
+
+        self.args = args
+        self.generator = text2embedding_model_generator(args, pose_dim, n_frames, n_words, word_embed_size, word_embeddings, speaker_model=None)
+
+        self.discriminator = text2embedding_model_discriminator(args, pose_dim, n_frames, n_words, word_embed_size,
+                                                        word_embeddings, speaker_model=None)
+        print("Model initiated!")
+    def forward(self, in_text, in_lengths, poses, vid_indices):
+       return None
diff --git a/scripts/model/text2embedding_model.py b/scripts/model/text2embedding_model.py
new file mode 100644
index 0000000..9456611
--- /dev/null
+++ b/scripts/model/text2embedding_model.py
@@ -0,0 +1,597 @@
+import torch
+from torch.autograd import Variable
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+import random
+from torch.nn import functional as F
+from model.Helper_models import EncoderRNN_With_Audio, TextEncoderTCN
+
+'''
+Based on the following Se2Seq implementations:
+- https://github.com/AuCson/PyTorch-Batch-Attention-Seq2seq
+- https://github.com/spro/practical-pytorch/blob/master/seq2seq-translation/seq2seq-translation-batched.ipynb
+'''
+debug = False
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+class EncoderRNN(nn.Module):
+    def __init__(self, input_size, embed_size, hidden_size, n_layers=1, dropout=0.5, pre_trained_embedding=None):
+        super(EncoderRNN, self).__init__()
+        self.input_size = input_size
+        self.hidden_size = hidden_size
+        self.embed_size = embed_size
+        self.n_layers = n_layers
+        self.dropout = dropout
+
+        if pre_trained_embedding is not None:  # use pre-trained embedding (e.g., word2vec, glove)
+            assert pre_trained_embedding.shape[0] == input_size
+            assert pre_trained_embedding.shape[1] == embed_size
+            self.embedding = nn.Embedding.from_pretrained(torch.FloatTensor(pre_trained_embedding), freeze=False)
+        else:
+            self.embedding = nn.Embedding(input_size, embed_size)
+
+        self.gru = nn.GRU(embed_size, hidden_size, n_layers, dropout=self.dropout, bidirectional=True)
+
+        self.do_flatten_parameters = False
+        if torch.cuda.device_count() > 1:
+            self.do_flatten_parameters = True
+
+    def forward(self, input_seqs, input_lengths, hidden=None):
+        '''
+        :param input_seqs:
+            Variable of shape (num_step(T),batch_size(B)), sorted decreasingly by lengths(for packing)
+        :param input_lengths:
+            list of sequence length
+        :param hidden:
+            initial state of GRU
+        :returns:
+            GRU outputs in shape (T,B,hidden_size(H))
+            last hidden stat of RNN(i.e. last output for GRU)
+        '''
+        if self.do_flatten_parameters:
+            self.gru.flatten_parameters()
+
+        embedded = self.embedding(input_seqs)
+        packed = torch.nn.utils.rnn.pack_padded_sequence(embedded, input_lengths)
+        outputs, hidden = self.gru(packed, hidden)
+        outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(outputs)  # unpack (back to padded)
+        outputs = outputs[:, :, :self.hidden_size] + outputs[:, :, self.hidden_size:]  # Sum bidirectional outputs
+        return outputs, hidden
+
+
+class Attn(nn.Module):
+    def __init__(self, hidden_size):
+        super(Attn, self).__init__()
+        self.hidden_size = hidden_size
+        self.attn = nn.Linear(self.hidden_size * 2, hidden_size)
+        self.v = nn.Parameter(torch.rand(hidden_size))
+        stdv = 1. / math.sqrt(self.v.size(0))
+        self.v.data.normal_(mean=0, std=stdv)
+
+    def forward(self, hidden, encoder_outputs):
+        '''
+        :param hidden:
+            previous hidden state of the decoder, in shape (layers*directions,B,H)
+        :param encoder_outputs:
+            encoder outputs from Encoder, in shape (T,B,H)
+        :return
+            attention energies in shape (B,T)
+        '''
+        max_len = encoder_outputs.size(0)
+        this_batch_size = encoder_outputs.size(1)
+        H = hidden.repeat(max_len, 1, 1).transpose(0, 1)
+        encoder_outputs = encoder_outputs.transpose(0, 1)  # [B*T*H]
+        attn_energies = self.score(H, encoder_outputs)  # compute attention score
+        return F.softmax(attn_energies, dim=1).unsqueeze(1)  # normalize with softmax
+
+    def score(self, hidden, encoder_outputs):
+        energy = torch.tanh(self.attn(torch.cat([hidden, encoder_outputs], 2)))  # [B*T*2H]->[B*T*H]
+        energy = energy.transpose(2, 1)  # [B*H*T]
+        v = self.v.repeat(encoder_outputs.data.shape[0], 1).unsqueeze(1)  # [B*1*H]
+        energy = torch.bmm(v, energy)  # [B*1*T]
+        return energy.squeeze(1)  # [B*T]
+
+
+class BahdanauAttnDecoderRNN(nn.Module):
+    def __init__(self, args, input_size, hidden_size, output_size, n_layers=1, dropout_p=0.1,
+                 discrete_representation=False, speaker_model=None):
+        super(BahdanauAttnDecoderRNN, self).__init__()
+
+        # define parameters
+        self.hidden_size = hidden_size
+        if noisy:
+            self.hidden_size = hidden_size*2
+        self.output_size = output_size
+        self.n_layers = n_layers
+        self.dropout_p = dropout_p
+        self.discrete_representation = discrete_representation
+        self.speaker_model = speaker_model
+
+        # define embedding layer
+        if self.discrete_representation:
+            self.embedding = nn.Embedding(output_size, hidden_size)
+            self.dropout = nn.Dropout(0.5)#(dropout_p)
+
+        if self.speaker_model:
+            self.speaker_embedding = nn.Embedding(speaker_model.n_words, 8)
+
+        # calc input size
+        if self.discrete_representation:
+            input_size = hidden_size  # embedding size
+        linear_input_size = input_size + hidden_size
+        if self.speaker_model:
+            linear_input_size += 8
+
+        # define layers
+
+        if args.autoencoder_att == 'True':
+            self.attn = Attn(hidden_size)
+            self.att_use = True
+        else:
+            self.att_use = False
+
+        if self.att_use:
+            linear_input_size = input_size + hidden_size
+            self.attn = Attn(hidden_size)
+        else:
+            linear_input_size = input_size
+
+        self.pre_linear = nn.Sequential(
+            nn.Linear(linear_input_size, hidden_size),
+            nn.BatchNorm1d(hidden_size),
+            nn.ReLU(inplace=True)
+        )
+
+        self.gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=dropout_p)
+
+        # self.out = nn.Linear(hidden_size * 2, output_size)
+        self.out = nn.Linear(hidden_size, output_size)
+        self.softmax = nn.Softmax(dim=1)
+
+        self.do_flatten_parameters = False
+        if torch.cuda.device_count() > 1:
+            self.do_flatten_parameters = True
+
+    def freeze_attn(self):
+        for param in self.attn.parameters():
+            param.requires_grad = False
+
+    def forward(self, motion_input, last_hidden, encoder_outputs, vid_indices=None):
+        '''
+        :param motion_input:
+            motion input for current time step, in shape [batch x dim]
+        :param last_hidden:
+            last hidden state of the decoder, in shape [layers x batch x hidden_size]
+        :param encoder_outputs:
+            encoder outputs in shape [steps x batch x hidden_size]
+        :param vid_indices:
+        :return
+            decoder output
+        Note: we run this one step at a time i.e. you should use an outer loop
+            to process the whole sequence
+        '''
+
+        if self.do_flatten_parameters:
+            self.gru.flatten_parameters()
+
+        if self.discrete_representation:
+            word_embedded = self.embedding(motion_input).view(1, motion_input.size(0), -1)  # [1 x B x embedding_dim]
+            motion_input = self.dropout(word_embedded)
+        else:
+            motion_input = motion_input.view(1, motion_input.size(0), -1)  # [1 x batch x dim]
+
+        # attention
+
+
+
+        if self.att_use:
+            attn_weights = self.attn(last_hidden[-1], encoder_outputs)  # [batch x 1 x T]
+            context = attn_weights.bmm(encoder_outputs.transpose(0, 1))  # [batch x 1 x attn_size]
+            context = context.transpose(0, 1)  # [1 x batch x attn_size]
+
+            # make input vec
+            rnn_input = torch.cat((motion_input, context), 2)  # [1 x batch x (dim + attn_size)]
+        else:
+            attn_weights = None
+            # make input vec
+            # rnn_input = torch.cat((motion_input, encoder_outputs), 2)  # [1 x batch x (dim + attn_size)]
+            rnn_input = motion_input
+
+
+
+
+
+        for_check = rnn_input.detach().cpu().numpy()
+        if self.speaker_model:
+            assert vid_indices is not None
+            speaker_context = self.speaker_embedding(vid_indices).unsqueeze(0)
+            rnn_input = torch.cat((rnn_input, speaker_context), 2)  # [1 x batch x (dim + attn_size + embed_size)]
+
+        rnn_input = self.pre_linear(rnn_input.squeeze(0))
+        rnn_input = rnn_input.unsqueeze(0)
+
+
+        # rnn
+        output, hidden = self.gru(rnn_input, last_hidden)
+
+        if debug:
+            print("rnn_input", rnn_input.shape)
+            print("last_hidden", last_hidden.shape)
+            print(output.shape)
+        # post-fc
+
+        output = output.squeeze(0)  # [1 x batch x hidden_size] -> [batch x hidden_size]
+        output = self.out((output))
+        a = output.cpu().detach().numpy()
+        # output = self.softmax(output)
+        b = output.cpu().detach().numpy()
+        # output = self.softmax(output)
+        return output, hidden, attn_weights
+
+
+class Generator(nn.Module):
+    def __init__(self, args, motion_dim, discrete_representation=False, speaker_model=None):
+        super(Generator, self).__init__()
+        self.output_size = motion_dim
+        self.n_layers = args.n_layers
+        self.discrete_representation = discrete_representation
+        self.decoder = BahdanauAttnDecoderRNN(args, input_size=motion_dim,
+                                              hidden_size=args.hidden_size,
+                                              output_size=self.output_size,
+                                              n_layers=self.n_layers,
+                                              dropout_p=args.dropout_prob,
+                                              discrete_representation=discrete_representation,
+                                              speaker_model=speaker_model)
+
+    def freeze_attn(self):
+        self.decoder.freeze_attn()
+
+    def forward(self, z, motion_input, last_hidden, encoder_output, vid_indices=None):
+        if z is None:
+            input_with_noise_vec = motion_input
+        else:
+            assert not self.discrete_representation  # not valid for discrete representation
+            input_with_noise_vec = torch.cat([motion_input, z], dim=1)  # [bs x (10+z_size)]
+
+        return self.decoder(input_with_noise_vec, last_hidden, encoder_output, vid_indices)
+
+noisy = False
+audio_context = False
+use_TCN = True
+GPT3_embedding_active = False
+class text2embedding_model(nn.Module):
+    def __init__(self, args, pose_dim, n_frames, n_words, word_embed_size, word_embeddings, speaker_model=None):
+        super().__init__()
+
+
+        if args.text2_embedding_discrete == 'True':
+            self.text2_embedding_discrete = True
+        else:
+            self.text2_embedding_discrete = False
+
+
+
+        self.n_layers = args.n_layers
+        # Todo: find a better way to play with representation learning dimension
+        # Todo: Regarding hidden size, sentence time length, etc.
+        if self.text2_embedding_discrete:
+            pose_dim = int(args.autoencoder_vq_components)
+        else:
+            pose_dim = args.n_layers * args.hidden_size
+
+
+
+        self.encoder = EncoderRNN(
+            n_words, word_embed_size, args.hidden_size, args.n_layers,
+            dropout=args.dropout_prob, pre_trained_embedding=word_embeddings)
+        self.decoder = Generator(args, pose_dim,
+                                 discrete_representation=self.text2_embedding_discrete,
+                                 speaker_model=speaker_model)
+
+        if audio_context:
+            self.encoder = EncoderRNN_With_Audio(
+                n_words, word_embed_size, args.hidden_size, args.n_layers,
+                dropout=args.dropout_prob, pre_trained_embedding=word_embeddings)
+
+        if use_TCN:
+            self.encoder = TextEncoderTCN(
+                args, n_words, embed_size=300, pre_trained_embedding=word_embeddings,
+            kernel_size=2, dropout=0.3, emb_dropout=0.1)
+
+        if GPT3_embedding_active:
+
+            self.encoder = DNN(n_layers=5,
+                               hidden_units=1024,
+                               input_dim=1024,
+                               output_dim=args.hidden_size * self.n_layers, device=device)
+
+            args.autoencoder_att = 'False'
+            self.decoder = Generator(args, pose_dim,
+                                     discrete_representation=self.text2_embedding_discrete,
+                                     speaker_model=speaker_model)
+
+
+
+        self.n_frames = n_frames
+        self.n_pre_poses = args.n_pre_poses
+        self.pose_dim = pose_dim
+        self.sentence_frame_length = args.sentence_frame_length
+
+    def forward(self, in_text, in_lengths, in_audio, poses, GPT3_embeddings, vid_indices):
+        # reshape to (seq x batch x dim)
+        in_text = in_text.transpose(0, 1)
+        poses = poses.transpose(0, 1)
+
+
+        outputs = torch.zeros(self.sentence_frame_length//self.n_frames, poses.size(1),
+                              self.decoder.output_size)\
+            .to(poses.device)
+
+        # if debug:
+        # print("outputs", outputs.shape, "poses.shape", poses.shape)
+        # print((outputs[2,0,:]))
+        if self.text2_embedding_discrete:
+            onehottie = F.one_hot(poses.reshape(-1).to(torch.int64), self.pose_dim).reshape(poses.shape[0], poses.shape[1], -1)
+        # print(onehottie[0])
+
+        # outputs = onehottie.float()
+
+        # run words through encoder
+        if GPT3_embedding_active:
+            encoder_outputs, encoder_hidden = None, self.encoder(GPT3_embeddings)
+        #     Reshape required
+            encoder_hidden = encoder_hidden.view((self.n_layers, -1, encoder_hidden.shape[-1]//self.n_layers))
+        elif audio_context:
+            encoder_outputs, encoder_hidden = self.encoder(in_text, in_lengths, in_audio, None)
+        elif use_TCN:
+            encoder_outputs, encoder_hidden = self.encoder(in_text)
+        else:
+            encoder_outputs, encoder_hidden = self.encoder(in_text, in_lengths, None)
+
+
+
+
+        decoder_hidden = encoder_hidden[:self.decoder.n_layers]  # use last hidden state from encoder
+        if noisy:
+            eps = torch.randn_like(decoder_hidden)
+            decoder_hidden = eps*10 # torch.cat([decoder_hidden, eps/10], dim=2)
+
+        # run through decoder one time step at a time
+        decoder_input = poses[0]  # initial pose from the dataset
+        if self.text2_embedding_discrete:
+            outputs[0] = onehottie[0] #.float()
+        else:
+            outputs[0] = decoder_input
+        # decoder_input = outputs[0]
+        dd = decoder_input.detach().cpu().numpy()
+        # outputs[0] = onehottie[0]  # decoder_input
+
+        #todo: it could be considered in the pose itself at the inference time
+        if vid_indices is not None: # for the inference
+            decoder_input = vid_indices
+            decoder_output, decoder_hidden, attentions = self.decoder(None, decoder_input, decoder_hidden, encoder_outputs,
+                                                             vid_indices)
+
+            outputs[0] = decoder_output
+            decoder_input = outputs[0].argmax(1)
+
+        attentions_list = []
+        q = self.sentence_frame_length//self.n_frames
+        if debug:
+            print("self.sentence_frame_length//self.n_frames", self.sentence_frame_length//self.n_frames)
+        for t in range(1, self.sentence_frame_length//self.n_frames): # +2):
+            if debug:
+                print("decoder_input", decoder_input.shape)
+            # decoder_input = torch.tensor([10]).to(decoder_input.device)
+            decoder_output, decoder_hidden, attentions = self.decoder(None, decoder_input, decoder_hidden, encoder_outputs,
+                                                             vid_indices)
+            # For sake of visualization
+            attentions_list.append(attentions)
+
+            if self.text2_embedding_discrete == True:
+                if debug:
+                    print(self.encoder)
+                    print(self.decoder)
+                    print("Check discrete:", decoder_output.shape)
+                    print("Check discrete: argmax", decoder_output.argmax(1), decoder_output.argmax(1).shape)
+                # outputs[t] = torch.nn.functional.softmax(decoder_output,dim=1)
+                # outputs[t] =\
+                # decoder_output = decoder_output.argmax(1)
+                # print(q.shape, 'qq')
+                # outputs[t] = torch.argmax(torch.nn.functional.softmax(decoder_output, dim=1), 0) #[128]
+                # print(outputs[t])
+                # print(outputs[t].shape)
+                # print("")
+            # decoder_output = torch.nn.functional.softmax(decoder_output, dim=1)
+            outputs[t] = decoder_output #Todo: this is the problem int <- float
+            # Todo: maybe we need a softmax right here
+            # print(outputs.shape)
+
+            if t < self.n_pre_poses:
+                decoder_input = poses[t] #onehottie[t].float() #poses[t]  # next input is current target
+            else:
+                # 0th dimension is batch size, 1st dimension is word embedding
+                best_guess = decoder_output.argmax(1)
+                decoder_input = decoder_output  # next input is current prediction
+
+                if self.text2_embedding_discrete:
+                    decoder_input = best_guess
+        # print(outputs.transpose(0, 1).shape)
+        return outputs.transpose(0, 1), attentions_list
+
+
+
+# Todo: new _____________
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+# device='cpu'
+class EncoderRNN_New(nn.Module):
+    def __init__(self, input_size, hidden_size, n_layer=2, pre_trained_embedding=None):
+        super(EncoderRNN_New, self).__init__()
+        self.hidden_size = hidden_size
+        self.n_layer = n_layer
+        self.embed_size = 300
+        if pre_trained_embedding is not None:  # use pre-trained embedding (e.g., word2vec, glove)
+            assert pre_trained_embedding.shape[0] == input_size
+            assert pre_trained_embedding.shape[1] == self.embed_size
+            self.embedding = nn.Embedding.from_pretrained(
+                torch.FloatTensor(pre_trained_embedding), freeze=False)
+        else:
+            self.embedding = nn.Embedding(input_size, self.embed_size)
+
+
+
+
+        self.gru = nn.GRU(self.embed_size, hidden_size, num_layers=self.n_layer, bidirectional=True)
+
+    def forward(self, input, hidden):
+        embedded = self.embedding(input)#.view(1, 1, -1)
+        output = embedded.unsqueeze(0)
+        output, hidden = self.gru(output, hidden)
+        return output, hidden
+
+    def initHidden(self):
+        return torch.zeros(self.n_layer * (1+self.gru.bidirectional),
+                           128, self.hidden_size, device=device)
+
+class DecoderRNN_New(nn.Module):
+    def __init__(self, hidden_size, output_size, n_layer=2):
+        super(DecoderRNN_New, self).__init__()
+        self.hidden_size = hidden_size
+        self.ouput_size = output_size
+        self.n_layer = n_layer
+
+        self.embedding = nn.Embedding(output_size, hidden_size)
+        self.gru = nn.GRU(hidden_size, hidden_size, num_layers=self.n_layer)
+        self.fc_out = nn.Linear(hidden_size, output_size)
+        # self.softmax = F.softmax(dim=1)
+
+        # self.pre_linear = nn.Sequential(
+        #     nn.Linear(linear_input_size, hidden_size),
+        #     nn.BatchNorm1d(hidden_size),
+        #     nn.ReLU(inplace=True)
+    def forward(self, input, hidden):
+        output = self.embedding(input)#.view(1, 1, -1)
+        output = (output).unsqueeze(0)
+        output, hidden = self.gru(output, hidden)
+        output = (self.fc_out(output))
+        a = output.cpu().detach().numpy()
+        b = a.argmax()
+        return output, hidden
+
+    def initHidden(self):
+        return torch.zeros(1, 1, self.hidden_size, device=device)
+
+MAX_LENGTH=4
+class AttnDecoderRNN_New(nn.Module):
+    def __init__(self, hidden_size, output_size, dropout_p=0.1, max_length=MAX_LENGTH):
+        super(AttnDecoderRNN_New, self).__init__()
+        self.hidden_size = hidden_size
+        self.output_size = output_size
+        self.dropout_p = dropout_p
+        self.max_length = max_length
+
+        self.embedding = nn.Embedding(self.output_size, self.hidden_size)
+        self.attn = nn.Linear(self.hidden_size * 2, self.max_length)
+        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)
+        self.dropout = nn.Dropout(self.dropout_p)
+        self.gru = nn.GRU(self.hidden_size, self.hidden_size)
+        self.out = nn.Linear(self.hidden_size, self.output_size)
+
+    def forward(self, input, hidden, encoder_outputs):
+        embedded = self.embedding(input)#.view(1, 1, -1)
+        embedded = self.dropout(embedded)
+
+        attn_weights = F.softmax(
+            self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)
+        attn_applied = torch.bmm(attn_weights.unsqueeze(0),
+                                 encoder_outputs.unsqueeze(0))
+
+        output = torch.cat((embedded[0], attn_applied[0]), 1)
+        output = self.attn_combine(output).unsqueeze(0)
+
+        output = F.relu(output)
+        output, hidden = self.gru(output, hidden)
+
+        output = F.log_softmax(self.out(output[0]), dim=1)
+        return output, hidden, attn_weights
+
+    def initHidden(self):
+        return torch.zeros(1, 1, self.hidden_size, device=device)
+
+class text2embedding_model_New(nn.Module):
+    def __init__(self, args, pose_dim, n_frames, n_words, word_embed_size, word_embeddings, speaker_model=None):
+        super().__init__()
+        self.n_layer = 1
+        self.encoder = EncoderRNN_New(3863, args.hidden_size, self.n_layer,
+                                      pre_trained_embedding=word_embeddings)
+
+        # self.encoder = EncoderRNN(
+        #     n_words, word_embed_size, args.hidden_size, args.n_layers,
+        #     dropout=args.dropout_prob, pre_trained_embedding=word_embeddings)
+
+
+        pose_dim = int(args.autoencoder_vq_components) + 2
+        self.decoder = DecoderRNN_New(args.hidden_size, pose_dim, self.n_layer)
+        self.max_length = 90
+        self.SOS_token = 512
+        self.eos_token = 513
+    def forward(self, in_text, in_lengths, poses, vid_indices):
+        # reshape to (seq x batch x dim)
+        in_text = in_text.transpose(0, 1)
+        poses = poses.transpose(0, 1)
+
+
+
+        input_length = in_text.size()[0]
+        encoder_hidden = self.encoder.initHidden()
+        encoder_outputs = torch.zeros(self.max_length, in_text.shape[1],
+                                      self.encoder.hidden_size*(1+self.encoder.gru.bidirectional), device=device)
+
+        for ei in range(input_length):
+            encoder_output, encoder_hidden = self.encoder(in_text[ei], encoder_hidden)
+            encoder_outputs[ei] = encoder_output
+
+
+        # Decoder
+        target_length = poses.size(0)
+        decoder_outputss = torch.zeros(poses.shape[0], poses.shape[1], self.decoder.ouput_size).float().to(device)
+        decoder_input = torch.tensor([self.SOS_token], device=device)
+        decoder_input = decoder_input.repeat(poses.shape[1])
+
+        decoder_input = poses[0]
+        x = F.one_hot(poses[0], 514)
+        q = decoder_outputss[0]
+        decoder_outputss[0] = F.one_hot(poses[0], 514)
+        decoder_hidden = encoder_hidden[:self.decoder.n_layer] + encoder_hidden[self.decoder.n_layer:]
+
+
+        teacher_forcing_ratio = 0.5
+        use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False
+
+        if use_teacher_forcing:
+            # Teacher forcing: Feed the target as the next input
+            for di in range(target_length):
+                # decoder_output, decoder_hidden, decoder_attention = self.decoder(
+                #     decoder_input, decoder_hidden, encoder_outputs)
+                decoder_output, decoder_hidden = self.decoder(
+                    decoder_input, decoder_hidden)
+                # loss += criterion(decoder_output, target_tensor[di])
+                decoder_input = poses[di]  # Teacher forcing
+                decoder_outputss[di] = decoder_output
+        else:
+            # Without teacher forcing: use its own predictions as the next input
+            for di in range(1, target_length):
+                # decoder_output, decoder_hidden, decoder_attention = self.decoder(
+                #     decoder_input, decoder_hidden, encoder_outputs)
+                decoder_output, decoder_hidden = self.decoder(
+                    decoder_input, decoder_hidden)
+                topv, topi = decoder_output.topk(1)
+                decoder_input = topi.squeeze().detach()  # detach from history as input
+                decoder_outputss[di] = decoder_output
+
+
+        return decoder_outputss
+
+
+
+# Todo: Audio ---------------
diff --git a/scripts/model/text2embedding_transformer_model.py b/scripts/model/text2embedding_transformer_model.py
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/model/vocab.py b/scripts/model/vocab.py
new file mode 100644
index 0000000..edfc1ab
--- /dev/null
+++ b/scripts/model/vocab.py
@@ -0,0 +1,130 @@
+import logging
+import os
+import pickle
+import numpy as np
+import fasttext
+
+
+class Vocab:
+    PAD_token = 0
+    SOS_token = 1
+    EOS_token = 2
+    UNK_token = 3
+
+    def __init__(self, name, insert_default_tokens=True):
+        self.name = name
+        self.trimmed = False
+        self.word_embedding_weights = None
+        self.reset_dictionary(insert_default_tokens)
+
+    def reset_dictionary(self, insert_default_tokens=True):
+        self.word2index = {}
+        self.word2count = {}
+        if insert_default_tokens:
+            self.index2word = {self.PAD_token: "<PAD>", self.SOS_token: "<SOS>",
+                               self.EOS_token: "<EOS>", self.UNK_token: "<UNK>"}
+        else:
+            self.index2word = {self.UNK_token: "<UNK>"}
+        self.n_words = len(self.index2word)  # count default tokens
+
+    def index_word(self, word):
+        if word not in self.word2index:
+            self.word2index[word] = self.n_words
+            self.word2count[word] = 1
+            self.index2word[self.n_words] = word
+            self.n_words += 1
+        else:
+            self.word2count[word] += 1
+
+    def add_vocab(self, other_vocab):
+        for word, _ in other_vocab.word2count.items():
+            self.index_word(word)
+
+    # remove words below a certain count threshold
+    def trim(self, min_count):
+        if self.trimmed:
+            return
+        self.trimmed = True
+
+        keep_words = []
+
+        for k, v in self.word2count.items():
+            if v >= min_count:
+                keep_words.append(k)
+
+        logging.info('    word trimming, kept %s / %s = %.4f' % (
+            len(keep_words), len(self.word2index), len(keep_words) / len(self.word2index)
+        ))
+
+        # reinitialize dictionary
+        self.reset_dictionary()
+        for word in keep_words:
+            self.index_word(word)
+
+    def get_word_index(self, word):
+        if word in self.word2index:
+            return self.word2index[word]
+        else:
+            return self.UNK_token
+
+    def load_word_vectors(self, pretrained_path, embedding_dim=300):
+        logging.info("  loading word vectors from '{}'...".format(pretrained_path))
+
+        # initialize embeddings to random values for special words
+        init_sd = 1 / np.sqrt(embedding_dim)
+        weights = np.random.normal(0, scale=init_sd, size=[self.n_words, embedding_dim])
+        weights = weights.astype(np.float32)
+
+        # read word vectors
+        word_model = fasttext.load_model(pretrained_path)
+        for word, id in self.word2index.items():
+            vec = word_model.get_word_vector(word)
+            weights[id] = vec
+
+        self.word_embedding_weights = weights
+
+    def __get_embedding_weight(self, pretrained_path, embedding_dim=300):
+        """ function modified from http://ronny.rest/blog/post_2017_08_04_glove/ """
+        logging.info("Loading word embedding '{}'...".format(pretrained_path))
+        cache_path = os.path.splitext(pretrained_path)[0] + '_cache.pkl'
+        weights = None
+
+        # use cached file if it exists
+        if os.path.exists(cache_path):  #
+            with open(cache_path, 'rb') as f:
+                logging.info('  using cached result from {}'.format(cache_path))
+                weights = pickle.load(f)
+                if weights.shape != (self.n_words, embedding_dim):
+                    logging.warning('  failed to load word embedding weights. reinitializing...')
+                    weights = None
+
+        if weights is None:
+            # initialize embeddings to random values for special and OOV words
+            init_sd = 1 / np.sqrt(embedding_dim)
+            weights = np.random.normal(0, scale=init_sd, size=[self.n_words, embedding_dim])
+            weights = weights.astype(np.float32)
+
+            with open(pretrained_path, encoding="utf-8", mode="r") as textFile:
+                num_embedded_words = 0
+                for line_raw in textFile:
+                    # extract the word, and embeddings vector
+                    line = line_raw.split()
+                    try:
+                        word, vector = (line[0], np.array(line[1:], dtype=np.float32))
+                        # if word == 'love':  # debugging
+                        #     print(word, vector)
+
+                        # if it is in our vocab, then update the corresponding weights
+                        id = self.word2index.get(word, None)
+                        if id is not None:
+                            weights[id] = vector
+                            num_embedded_words += 1
+                    except ValueError:
+                        logging.info('  parsing error at {}...'.format(line_raw[:50]))
+                        continue
+                logging.info('  {} / {} word vectors are found in the embedding'.format(num_embedded_words, len(self.word2index)))
+
+                with open(cache_path, 'wb') as f:
+                    pickle.dump(weights, f)
+
+        return weights
diff --git a/scripts/pymo/__init__.py b/scripts/pymo/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/pymo/__pycache__/__init__.cpython-37.pyc b/scripts/pymo/__pycache__/__init__.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..81b0b25efe59260c08b5ce0d3b3bf936051ed0fc
GIT binary patch
literal 211
zcmYL@Jqp4=5QR5jAwn+T4J_hZKtxQD&L#&~Hk%<?-9L6WMZ`;Z46kHi;}PtvoPgqk
z_swHq9<$H$1tUF{*Jhf}2|r~NcH}rCw5(>U>~LdYTL1CBfOb;oBFZ3=?7<(M!BLzc
zxTuhK7}F4tH)v&VL^y%e&SDFt9%90vYzKxCl&+{fqU?F~J*m(lbt-3p%9T$M{Ae6-
XuB=L&ul;zRZ;WdPjoay~7nywka(q3j

literal 0
HcmV?d00001

diff --git a/scripts/pymo/__pycache__/data.cpython-37.pyc b/scripts/pymo/__pycache__/data.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6e0564f6fa9de55b69c54cc7b447f5d22a673d40
GIT binary patch
literal 2748
zcmbtWUvC>l5Z~SV<BOZ5NhpY<2&{sD6K#y(1*EE~NJJ$h{6HmIMXFBM+qL85e0SNs
zl-N2yr4qdM8#Ir6rG4Oqr+$S#F*812V>eVnoVBwvce}GQzxlKNVQsC=;Q4Fk#nVln
zvA<E6KMpYWVVEz0NG5s84)_Bmg>;@X>70uL0g@{{kUWqa$se-Ne+8>qFF`tg8ciR-
z8wFY<I}orFHSv0%XT6{DVOI7Mx88=cfUyU|;LeIyEMk=a$}0z`kPJ@bz0k347LSzm
zi&(3yv_Wz>OeL_PYh9z#zD5%?YSbQ(Kbda9^!luu=1H91F^P`L<gi;D<s&tYCtaPJ
zp^65|l&4xnPgRy2j$(b>P0~D5vb&d_no{XuHt3Fk>+a=u9u`U^htY$2%LA1u&^gSr
zE+`)srRf&qQQnnt8SfTj8${7C8<tUoHW`4PG48`}vQ8-M&Oe8u*8?YDIXW0cwjD*I
zT%M-LuSU_yX`If!=oQdGZe1b)zC#eve}hcGNlt)m9{hM8hWQal6CW<ca|Y4bpcwN5
zSNgIAZ#)^u6_|b5me*iz$yK=qb0F8{b(mMA15w$uD?bCa#XdNkaL|hn&P_;ffiPqk
zQi+rgkP3-Gb|ePbrSLkv0dB>|ve2`EIaaAE^UV6s;`CG*yOtcrS*B8g$k>&>2B-jh
zWmj~bm-IU18bMx$1~J15FjNK<0gA#_V|x>cUV@h6gBgTjLktPBiXE{5pNg~ZDxvRJ
z;w8UeQ>Su%1&7Z%@KsdKHWbh{+h<SKr!K5;D))kOc5?d>J83@x1OPJ8qvQ--{T^tx
zPLUV-I&ywo6e^RUM?P47WIY3@9$W7SOUCkK$uAp}I)0|KQJZMvDiFp4a1X}~KDp%H
zd|?wKOa>f#FwA2h63R7YQ(p3OHWd|r39&hHE_fv(r<TAMklb4GNb+jQC&{m+7D=s2
z%>2cK2m|XTc`?>&AlN`ERgj3T>kTC61Yx4LfD4^jC~pF9gG~JvVNoq3;8Ce=!y1D+
zX6~=^`EPRlO@wJ><3C0jqf7pO1`PNQg@7^MXXhMocnOV*-f?_$fo;#7?Ej*mS!AXY
zr)ekecgjQ6sT0$5pf4FEJAIvx=tF;)s!j}RJKyYgveQv9?!;Qh<F~AM2nLuNK(IQV
ztWJ3a(b#4$#EA>1@>d8TxgH8DcE7f~zXZhV_T6-7$|N5Z_dZ8u=%3FfyKnp-s4@bh
zqq()acd09yd%(=MUTxUU+_0@Fm?b&DM3B_xiK}<L!yPoPc1L%Bx2@W!C9m`?`1l2k
zxjWzgH+L?1ws4#`>m7JG%ub{Rx^x&MAf}WRSz%kTp}>V=R+~O4FiYmzjb)5KMcXmW
zn6tqrJO6Hd10gPU>OoiX0BZ7g*5{O;!5?*gO6W9GQ9?aJt~%6L++L#u@kK@0U<|rN
zX|*%uxV?e~g~=uPvCvS=^C;52*@F&>6$y@ZVA6OYFErH#sjuM6JS^Qs$L6g&Dz#Eq
zrJ1INAm}8zr=S^P-?lF9umSb+V>Lc#6G!cD7Qni{qZM`&(Ox(!M4M0%<1YVDOm4oz
zt_Cc$1qXEkM2*noy@dl8Q^juK(P~tb<EEUtu#@m$V)D`Axr+<`*TL`O!AYLM#gG9r
zyAjQ|m(KoV@$9$O;vcED#*0v2LxQ))>`w)jx&9al;;3&Uxr1aE$r}WSqddS?j(4kB
u2NLk0)eeGo;KR?QHV=PkJZgIydlE&O-e0B626t!wc<x~^sAvMQ?*9XwNK8Wj

literal 0
HcmV?d00001

diff --git a/scripts/pymo/__pycache__/parsers.cpython-37.pyc b/scripts/pymo/__pycache__/parsers.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2a2f814c8c4822042f0ca647d20cf4572b901157
GIT binary patch
literal 7821
zcmb7J&2t>bb)WB@9V|c)ruikA9!sXg6-9uO%aX$|v`7#Xsgf5m35(QH^mwp63oNkM
zS@aAfa6OxYs8r<;SLCu!E+2@SQ<c9Ul~a!S3sQ5)B_~$pn3FE?@4cR#U93QvUevtq
z>5u8x@8kDgH*U<%dK!NJefGEOU%sMg|3fdMpNYzQc#@kSLThS57e-fa>RcL46Q$X;
zcI>9T<1`&zdtVclu)oxV{lsWi@aBjL-YR%=g>g@-yMw-W<Kx@@N1;q2=`VM~B=Ni(
zkNwr1j_|K{qn(boETb@u#1;R8y<X(M_4b?o+n3*Z7o)-?5`Nsf;%}yDKe@8Bw9!e@
z#f?t7xwpR9ig%W_;>})LMxqr(OK)Gk{O*r0|H$*6fp@J@H*))fxYJAX>d)d<*uNR3
zVT*c3KLeHb@FYG+q+yR*(-1oLXbAI5t!b&9HErQwAC;W8ajzBjdXXGl@O=EP`=86O
z-^aFEv@O3KOJ7Ed#ULKi((!TnIe4mgmhmJ_5R&MZ+7R*-7D_{49h9c12p6R#s=`BQ
zr`l6p%w&4o5Vgz%oz3i$&Se(r$He?b<v`E0zz`aEJ>F>1=IBQyJ@@b=E{M?SQy644
za{JEBmB!k?pq^I}wX@t#<L#&?iCrdN|9Ytl@pUgHEg7b*&87ZUyc0bR2TL+eI#I9@
zCF!1wg7v7^+T01{_EM`G_ad>h+}%skNOpP~OFO77EytJc^`oe@8N6Ric|U?E!?Y9k
zmhkaTKTVeUk9XpwKATFi*ngaR9TD}?PP-Gy6SM=~mOglWtct}T==3^i5G-D0K9g&U
z<2f%RhNipv(aPQHYpZv8!*xSeX!9hju^1%jE_Y)nJ4|DFYJ!>b-^$Er#tX5M+c)pL
ze`oD4@?nb(oZ##P6R&(L6FmIIIPf5hMsVZq_2ren$cBf_^@4R7wxaJ%@bc~28P9n!
zhGv#OT)nsQf*j4j_!1Z+vE)y%bm3dYGcjpUdl+6CT))%|9)0@x8-rIbz47q!rN4Rf
z#;1!v<;uki52KYw4=-JORJ_-Jn(M<{|8?E?TE7C&8JM3Y7aMgmH)NDo3MgQUsyn%r
zMBTO|_?0t6W{Dgla-2x%bcMBzOnU*H329q1Tyyki>5k>->d(*z-#KE3hXhQ}&frr+
zCDXP5E17<v0||7n%^ZS1pa5RTymK3l-EL1YCC{MiGM<qu9-CN~W;d`+Hwb#+P80;W
z7X&-8*y~cg76iL{VYle11c8WKK_E}#WBk>Ll~@quanb^rqvfc{H7(se$8Cp@oVNBJ
z4az_cA^G<iU3FkNqbso7XI#~rj+hZOpnOHliaC@np(x6#I4%}Ydg43c1j-rlU2zg+
zO`H;^QO=4p;w;KJ@sfBM<uUO+aSr7?N4p>7Rmeldy1|1YNT}eD#y$g^ANm;Lw>#Zv
zkq0nY*-v4!Y2v4wk)Lcw-6)NFe%KR!Ym-7w*Z(Z+?nMa?{wBOnDE6s#py`O+O0jxw
z%P5J`ky2e+msBtzZ5A}C;crQX1;j3t`zR5%jAazlXc(`04V4ZID$q8F>P!u$$QICI
z2?Nv?Ca9BEp3o>at#auxpH@TCCpYl$FW2fbc{Lb`A+N0?q6DqDmqz<(J`a<kpb-?3
zNnmfPCCO)7dr~4|1RTZkYFpA)kzC|e8OJG`CEJg}cru*a637hdIvf48RD2Ae4JPD*
zTH$Sl*BP!cAnRA!USu)zKA!zQel4<G`cWJy1OM;wl4(jsDg009PmF(_8(Nw5)Hpz}
zzYoRzT)V41z^E<rsg>CW`lrUw8CEdM&YS~XryfTbr$~$rS}U288JT%ta*vDNYT$v+
zaIJ<Bvj;S%8q9$n;~LR<t}%tFn2lVQ&qxdP<I+cwTLjd(h15(aVUg!U_-w?%CL(eu
zTbqyN>(o43G%G#GSJ5WlpbmGWM0p;s&Dla{tO%$JdI~5pyG44+9cAOPPSY=d)Lo^O
zAJC_ON1;rCQEA-I-Sy8l1G432O@^N<+vuf9?y%w$s5{yTJ3SG}DV63nZ{S0GQ9-Ki
z8Z~`EpVvLTrq3B3m!OA~%X3QmCoHn@u>H4D;p`Z>T&8ao`5c66+|-`vggd`6klj@%
z%Nw;0%@oQNF1ML9?m*{^PmQsuWoYATPG&*5w(x#qLP;}&TX!R5TfM{&yIntSGg^$?
zK*5Xf+cMta4ib+aVk~-k0+mwIY^IHJKjh0~uAC%No<-jKs7Ct{+8OP)#;!$Z!n0VM
zNEnU9%eg)s9lrLic9EeYGw)wp$9EV^9BpuM>?VgaKF-<3^Kw0@>$$aur0%a~@DkQ6
z7?cEN^6IETSpg&6-=VFAr(o{rMJU>`6c&)7L%-0SPFg8sVH`c{`LcUPGM;2DuM~Py
zoQ)$Mu_wmt4c_@HWCwYU#D8clTj3xt^4ygUZHY#)IrylsAWoAY2(iu(HO^>l<K)OR
z%z=GA3C<_VLZ-El>#d_U8I3ntHx*jg<gxn@P2Anv=_NRHIm&eN8h|40>_j5!0tH^f
z)ZFOx^GcY6QihKiUGi#a;B`mkFh*bG9?iocfxhgpAP_M|PLtQvS_+V++58dK`6aPQ
zIg@6(`dQ$_34PueoO+H{S|*LekNk;*$QVI(ks&f}z>@G7f3$~YMjmZ8<SXc_+q6*%
zT(W`Jyb`zD2--<*#}W`m>~^{W{YtBrLdB10s71j%w<tnRagPYvi#`viXA2_}4oKji
zQ^k+?jI~&fI676F$2?$Gkt&X}3Pb*o)^_Od+Y?1@A=l70&BH>_-%B=`iBSOW5i{ft
znm%}WLX6)SSQ;dSC6en{mf#8D6BG!mp9V%jmPSL8<$)V!&P1MbbX3oe<J6iS;%OHX
zM?AeW!PE4(qgkT$h#3Nke}HEoGV^=MqEWC2bzj8TNj~K##PgBa2l(vA{X=}F#Am6<
zNlvFnmWD`yie|9z9-xxx8F&=>wkbJj&2ZguiY_3m4TP%`xPsV!Go=e10Nx{iXi=WR
zGt4s(cV=eRRc$n9_s#nnS`5BXaOfOC%Qc<VTet45tU)D8CuL^$jMPXsGcxRLM7iDW
z#$j5wm9v_N>G=!?aOJM@=NO&qooS=01(cNp%1QPyCLu%B5WjIgK>VJAkJj|T$q7YH
zOeiHo$NW+<|B9FANhTfjums?`6B5fzNDNm+LwkQ6k}_Fp9}k_e%n-tzvBc;N(zmyb
z<R6Q=Gf}T>8}bjCUG%tTw0Dse?%t;F@7{*2IUQ&dbW}@eyUVw)HySG+-YezJpwg^9
zka5cQE6wUB<7#tMWweGeOk9%5N7#S8qJSkpUH%GWoUgUwy#k!%UFr>qa42Cyfi<5~
zY8gT67iddf1{o=4o|R)kJyiRlzF@fe;PkYTM$=0*5%!kZ)>ps+p+g1ABw@%%o()Qe
zhsH#kUAAeIY(pKEFd^F8nIkNWwFMzC;hKXK+dz7)zDKI5z%f2c4dJS`Cx*hki7)c%
z2L=*z7puBiMO2|+Gs*Eqi)NwE99D<M(95dm_0a2~oVlv)znoR!Rwss(p4(5cdN$KG
zD6QZ952zPu{#GTkF`i1|rglc7Q8Z$IYl6$!ZR0Clo-10&q35V2x$%uL>l0%j1I$_n
zpRW2QYE|lo3~)+&>e1P8_iJ!6md7!k^Y?21_cTamUiFS7gZsu?3~9N!(i2T9XzZ=8
zsZ@ZD?t`=Uk$Xn_{iv1VHqgiAOs^Ha>2JiT&mPV(Pni{^91J12yIsB0Sd-+~gX6#G
zsp+^4W_a-HGm`F5<;Nh%=yA)K%5`cGM9verPvli11bVoA;2b{KqAOwf5KZ;kw6(hk
zW<k<Pqo&Klf)26=RpTf-qhd7Y3c0PmL=ZKxf$Q{o*e~QJ+J1*8p_8NrjH<!>@!$aI
zqDJrYa0<dMatsTn5o-7J6F7~WFh1ZDNzFj3aR}2UHdKahI&GHD@)|n}M<WiFFcJgC
zeVPv49It@;hk7mazVqB(WPZRcQ$FAkqb~vz*>os7Br}3rArd;pm$j$lmS|yju~cMy
zN)DvjOdN9%Ey+PMTz$jvZImub2PL}|M#9C(C2#^}SAi4Zhy*7hcX4n6HnK|LMEQMf
z|AP8DxCL**WAi=BCSR;dEAIXujGQhes<fN>jJkpt1+ePvJGc_QyL|hTQLHAR=GNWS
z)it(H`5xY<t+$P~y2mGCw#*5WEDdqOILR*ftUEg9GVaphx2bL`IH`L91G|hV|Ar@_
z3f)qWG2t-FbV0X_IlR>n6i>2Yk!g8sS|?#%gR>I|tW2sr+_yN@z&Gi(k}#f=h<}G!
zl_DmiAfq~$;^g}TD0Rp<tioijqSq;V$v}X@Yz!kaqLGBWzbwtK7?BnRgno>{xtI(^
z-`0|UqEuwtl3!&Ohh%-*PX3shUjVr>6a9bU`t#OuvDSZ1uT}1fxT#{q|4xrc3E%O}
zk{cTai`y+~8|_JWL)<QIP^yH)?*#4BLCmV`f00&-1mUxxyP*xM`!_Jpg(?4pA_b7x
z<KvY_JDpysZ5Ee5<vL&KyA2qzn;F|SQ2iQQrrL%{<JN#22~h))4s~H6iKt-oHM(Ac
zvDcKbvmp<TO@b@#*Vy2j&d*lY?yNQj#~ZP)E_M8b@H_4`9lor=*+gC1SZg?5l?fhW
z-^d53RklF#x$6o3`$yAPq>QDcLlxif@KKJFkFjF_-01>ZvOD~kYVXiDX87!eQv@=R
zrl(%`YOko%F%P+n*yrS=IaeeSL2;>c6f(bs@BSB_<O2|maaW-)xepwNs4))*g8T3-
z(PX=0)C>!VO$iOPzlO1tdm^csJRK^RQZiIm?QkEB&jGk<dx`3lBuV-YiabZ;G!dW3
z6(V#9Rd7OqOvOElUlZuUNJ7ppX-5<Br%cY|_t8PJ<c!R<E!{KSId8$OxfcIN!E!zK
zxcfS;*J)(Es#y6h>bc$9+vz`+w^46a=|6OfbYm*%2q#S<gc3aMsFic~s`?keH3E>t
N1EENVSZB<={%3hGpDzFa

literal 0
HcmV?d00001

diff --git a/scripts/pymo/__pycache__/preprocessing.cpython-37.pyc b/scripts/pymo/__pycache__/preprocessing.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..80126ceb5890e3b284447a495c1191e103621a83
GIT binary patch
literal 30154
zcmdsgdyrgLnP1;WKc}asr!|@xJuDj8#vVnM<N<^O!br&2;KhTn1h%_YG_Afpqn_#M
zZr$5sje2{s3zip|AlB@X&1*wOZ1w>a$nGX&H)KgRFW4oaQVIDd_Yaa(QlXrJ6qTg1
zDQYX^_xsL$cK665$=+1bqkHbT=bn2W-#O>|e&;*iIePEpWG;c<_x6A8V)N69#DC<2
z|4YJi8kb{fiG-D?BrMI+TUtevyIwKmZdQ_VPgT-#&#Y(lgqB#(RdO1?Fj^Dq`O2jD
z%~oN3sxq}+tQ57x6A3G6rLHBcRAZ`9e9^2-BP4BQ5R&miN(jkXIfUfAkUa>Qu<{7W
zdm%FjnY0QBDR?1!5i(^J5mNL*W)U)Nl@L<$Lgo;%$C^RNj2AMGkiFI{LT0^?eF&Me
z<`FXQh3rSjK5IWh_In`*5OTmeh>(L`$U%hMV;w@sAur?}guKZrBc$ww974#Ot-}a8
z?1j9^($6Ip?(P44?rFQxwLA5O<22hV<@0u}z1*?a8@5w^Z_Q~~<xaco)SC~jHG2;@
z<ugxzAeVdZ)$)hdn^yTutFhk9owXaao{IP6M!Qjdc=2d?@x;UL$mO2uSk2`oUR}Ie
zKG$4FLil2{*0zv%r_ET4CmuR+@}b4I<z8hUSX$8B%rl){t=H_d7j$=$1$dv+Yp&ON
z9osFkY>q12Q_an0yUvgNp8?NlT+VqojRZPPqN0Q3x@BC`&~;{{@wJKxqO;L8bfZ~m
zY^{<~O^gmD4O_{m=B#8@BUW;j31ZE;rKdXeTKDOiU0ZMT8g{eau-#;1qt&qMI&$TY
zcV1_4Iqd2KZIGy<Um^j$e<4@)eu1aU)T+I!-A4J{<)3*#QhVTN`GLzDwI1({&F*@w
zOJBF+G)b8-bP+)hYz0Y?ez`C`P|$Q5tz|onVtgrKXL0-SCyuvJht@+*-LCcOmyUN=
zJL`?Bwf=D%sW+-C4X3wZH>wvK?fRwln!R?s-s-d)*738g4Tn{3uN+^8@A%oyL+84U
zM*ULt3Gb698tn$sL;;T@@n*N@9PeIT?;P(&``5AVRX0<uHrvf!wYrBi62c|>2S!PI
z%#XKJXXEn6J9vYlt_jZ|0lI2SQnv-e&LGIuo^!RU5~3Wof@>@{d;8&0`Dy*xG5H}%
zEGOnB?bwE=hc+A}R<*gri<*XfSHYXsbo<}{Eh|q3o=yxB+xkGe>4%vJdo+lV#M3i~
zVMJo25VnjkQ-xXSPom?ZPMM|t%xDqcS+-NGO-l6&*JLkoGk(vg<WxBOMgOGqgfe@U
z^UkvLtD|McKU!9?j+XuJ{1AS(kaUw>v~y3ojhn0XI#n;XH2sKDDR?2?vqyM4kD}R=
zblBA*#LxEpk$VqbIP-84T0zTcCNBQ@_nh88I9lst!RO<f=YvQNO@IahF$*5K6Vg0~
zL0T;!cfw*LEn4O$wf3xh!6tdkN_`S>&OsOi#1}H#Q%Kk@!s#C%;TDC6<)%|^cY0-K
zqucG+;C%KpW8CW#(<v`1YC9t_`$u-)CJVZ1fMSH!l7E!iu37JNNBa77v;6{CY@-^`
z?HEev%)m)Rh@BF@srT<2L-LU%0uILpX8X#>SjA`@XxoXKi7maSt?H}j+my;T^ew|O
z5Ms_G2F841%Y;8UFf3EvB?l&AC129_;Vl_6eTbFnY1_QggVaq;?#Y|lTH>Z=nfnr3
z=|OrmL!V{LN@H(j1{wP^gS4N*77)Q8y`2r??MK{Y{U%23R(A9Ed$~bYrSPKmSmMH$
zw{qal*#X%5GD||AM&C<1Qbg(KpGcq_69Z$Xc#LDEZ|VmUTN49wd@PhbKbVko@+xI3
z>3;|M=;Ocg%Wx`j`S+ehEOLeZiRI>I8S?rfWUo`c((GL-TTQ3lY4@7#jn0Nswmtb-
zexXr^)O21Yzl2~gYnx5yRdQRKAxp3R1V41LeRz14oNgg+&*A3k?XH_@HQTk;ifgt3
z1zf$`bWMozWjA@{Qp0YzM(v{GCY?*QZo^I64dqE<jIT6Yv)61j7A6%dHc=X<l4&=t
zRK;f^3Z84NUR8W4OWo7zzF0AxD_y6OY&spws+izHFSvTMSxGU4+NMg0Uk-Da9LT~f
zQ0aG}63!tw2|Z&JwSrO5=Cwmc2}~xh9W-XNCGDVgTJPU8rq_q51l@fv5~2VliJTWe
zzZG=#SM<%*0lLsD`h|~g83-|Pp?j}N7hcr|sa1g08yY$!@kavB=3@{XTj;*A=PbJ}
z-W#h)d7o>d8@-}!{%DXx>e_|>{Ia~eo?s`FZiJZfl-ZYfSsSE2l~~q*VvK7T3QPfU
zCcV&{gl4_a!eDY0I0xzE*An*kfOJ@{$?f#bgycMB6<*fb^Pft53PS++hbczh4hrGd
zuW0sut0><Wg71s%St-l3d>IMvWmdD-wZW8Cvi96CwsT+=(<`9jiawaQo;bz&;l8FL
z)fWcS3_}@b5MK2B6Q>gGkMYeQ&o}!L7&4Jqz5Mnh=_lXq-3MlY!CneuGso@1pmbB)
z+T(?S3WGg!ekw>G%xMZ}kQ7lVBL=AgMq*fVH*~O=`Rj>UlvZNyyJgJ%k(dW=8S|iu
zd3hDX+t@CGGBbnpV5W+}zCAsdxe1_gIqwbqd-|u&cF5OzPI<M1kr57c2%2{JeT1Q9
zjEz>qDfc=mP9WhOaDZeJ$a}NS(a#@Q0U^{mTq2TJAVe<|w}3we+FHs6BKe$tnZOsQ
z+~+DX?f@+HV<%j#J~Rq{oIBNOI=y;lz58gLsWOXHqJM16Nb{h>u~Py^ok!sbKD~ny
ziVXG&XH(kQ$LYjzh<yTK%ecIP#Yn-jI=o77K#1HHFu_NGHZSV}$b68gOY&|~+F&Sy
zA7@Xk00nP$?MIt_HfEy#(5~w5f9a+G5*qEEo9x(_Da8uO_7c=g2<5le^`Z6(3kCKb
z&|*QeAI3-OB_Xj;!-i)ZHD33OHo}ZV<k&=lvAS{gFMV+q9;sV_eUX1FhTjuKc1az)
zy1CkNSYJD+*VQn)it*g@l~_*f6+*>l{G6U-V_RfA+V#YB;Fo=gWqlUUu~0mZ2{ruC
zpp>T6us?afQ@!5_Mh<(meSi-0_nYSJ%Fm{8vxr-jMjvXW0p6-s(qk35Gm_cNa~#n=
z5o8l@8Hzul*E`{nmRXF_D?%ro3@xCVp`M26k_TO$^U-pjPtV7mD*n?bp~m*%7lUH!
z=fSJeR%69(G#u$FZo0F)>@<2#jBWwjBP5mz<^2lXqN6p&(QG@&98C(8RO{93qX?6F
z1+~2G+=<F!)&H}8fGR}L(Qk(N$4PnHS&nqgowEGmP?kY^EtK5U3$@lp!vSQp-0Wi@
z#^cnL2u0j<z0=xQZ#!<rS!=W!y-wS{A4$2HCKL;H=ZfQM&$zi|yR%-EiH>Vv62d`&
zQPE~91a)q@=G1JvcGXSRy4}`QH+2qL3FMd*MTMI}XrpZ{>=ghiQ&Ly^u$#nGsJ`ZA
z>X&NmcBAFEsa~@ON$F-JtE#nJNuk^EPFYt)QGrt#LZ6#b61RehY@<<?Mt&;`quX^<
zwO+}qR8-|FIezE)(?|{KE7Wbx)h@eP0$C}G-@3w-Pa-kJyx1c@ipwEyN*E@l$tM0%
zM!_iZPdD`f!V1PL{t8-2OX+$16}1^Xr{|3sUDt9(9!R%f%;Rs?m^Y7TN3>%`MoZzl
z{{Ee1Q2=>4?VVumxSVb9ErGn+=H~_SY8O5|40#8sZ4KZ>^s#_Ysa0JdWyZpM56{M`
zA<tO=+qVq>NC9k*0N5(<nzZx?c+D}kK)Dp4?1YtubepgMcFB2P*1YGe_pE!*Iq%s3
zU^B04*Y)ehiUC**&Fu|C1El?w{wZVG0O)02x$UF}dh@*$=9-D^OfS2d8%%6Z+)NDe
z_^JS)n!gDs3D7u+G$$>n1j)^M1pvh&QUP@K0AkUazM)$s=HtcO6U3Z(jhK6bn6o>@
zy!^C)!)egB0B~3ka9A1?V&HJDzYu`KVIcUv%`P-}m><?!<)>=h9XMebZ7rN|J_LZ6
zOlxcD2oTimhY-y*mNqW>-8aA&iZ2P^3YgRHvIBV_@ZlXgp3*J7CJvOo$mFQR13W8v
z#Z24Kw|OuPDjbN_k064l5cJ?rh<n1~Eh5&}nI4M=jCNwT#q(j>2!9m``Fi29Z}>$^
zCG5AsKNd`gkRaPWk7|=HZo1j-wrcf8tZ>vAO5xa6(#CJ`r5y*Ff;iVPbM$-wKY_^B
z6j&QUEQXu1A0}_aebKwWmG>d>^iPqc*8>1R-+`xo1_FPjw!U7A6>>KsdDjx^4<oti
zHc^X6pasJ<Fx_xdC?^Dwn-m`1PoqE=8@<~v&#r~^ABE+S>epJ`OShk$0LW-d>?EMS
z9cH(%$9@;9PtA_~D4qAf0Zy?yUHdejMW`l4q}qRm0cYsE7mk~3b*^BABU^Jk)ry;H
zwjIns?Nf~N1RY`>`~7r20B2z;6ub754E-rORM6T_(OH5c+G|K!dx0SzVu)tjA_Cb}
zD<)WiW1r_ssjaCdF3&RHIXchN5hlZK{bgJZXLeC(3C#~=WkH*T%q#LwpVjtj_Zbmc
zI9>x3s=H(2kP8dKg1VtNv^;TGxeIX!g&`D?f3l-6ggCQO6cQA4a5Hp8wB@V`5pC4}
z<cGu>)RmqPoutl)LavvAT6E&31~HeXn6q*aa!`zJP4@B-Z&NYxQ?QDn^O%P4&08fU
zT%r45OsuFygkL<SCzidKap4z_2|W!3I#ioVyiLW#TX9Id%`OGJf`ecTLwgzPGopUN
z4DMRS68!O^W<%Kq)e=P5^@IoP%^)Q*el{tAoWnUMqzY!71Iz%|QaexlwJsQ0s9cb4
zNMZ6lCgTZ{GvkDb5_HnPsHzz!@s0)R1F)*3a0%BWYjTsqF-5jvieuE;S5XR)y)s0+
z5=mh=6oSG)`|^%EEX)X@9M<r46u<u=La~##SI9pdA^e~G0)mQ^TGAkaxCjNA$VguX
z+eGg%+P%)x=UbLUdV>dhM71z>#{B)TVuCI_(HtxFZUiz1V+;uHB$0j)W(cSf2;_07
z=C%v5YbpH0un@PM$*!Tte+@GcWzWL&I00;5AmvWeq5QQ!M&}G1cS7dBm^5M~!_7eQ
z%FNQPGQu58-HZI1qPM~Y;;Jk{IwE=PI$v?ZCjuDn$(v-cCL)2oP9*GwobtQ4oFi}&
zn40<%R3&or+N^$9#O@&_Y;~D<>iv6nF~ZRg$=k7@@BN#gu>j}()hbq^tLq&L#e+Xz
ztztp8rQX<_tJ(!Rd+88^D=5Rsv`vg)v(%9p3Wte85LBU+a5+Ud87-I28M*1)WadET
zd}hJsTT&?aQDB_L<q(tyRwj9w=+eq$(n?ur*p`@90aL~-d`ZYw$ZGgfRuU7GqMLfE
z33Z1px{33#6>W--vrL_ZQHCQ}986TPq{^vH|K7mVLnQ%4|D{Hm>WFgiX33iiC_JW2
zB4lzP0TCKVYm#(-%p5t~6J|>5PpPEu(D-6U8vtpapD-s@GNJWzUOrKy3pO9QrO@Cc
zXdO4YfEvQUsy)msHfM>wAg)cUo9F{mCc}rG#d_T4|Mu49{)bwV!#620&!Dl!$9u*$
zAc19C=86IAgQ-8vIA{aVmpAe*@`lD4pZDJ2o%hCEO>Sj+snzr#gT>@DR{644W?-&n
zXA@gFE9?0tteocq{sQVsbM0*+F@X9(R<O7bod?RoqBHQ4iUC!!#6V1}afLCsLZxB=
z&q)ldgvDayPr)3`ngTM*tWJosWfE$NX*`>&d3i2i{Vn=DH88g)fx3!N%FGW6qLe9Y
zO%JBG3p7-_sc)6SawC<bwRcdWayObU!lq?aEGWOkoE;N$HWm~0keG90V$Q{4a;;9~
zj+{nvM~MR3fO=l;qcNpKLCi4s(U?-AQ&3S|ZaoV_yco^)Y<_022mS8a*31ZPX2z5b
zVL_f@=|*GD4x%x`(v8NPQ!!uBFaI*sLfJvFidAQffTE0mN2n?Sr`WgD&#2)AgP}YI
zl$ymKgwNHUceM%_EhlbqC3ReXEKh-ISCr{d(JjJlycjJb<z1?h1klD1M2M_I9!*4h
zFR+HENPxYHFwoE}XdIS~m5pp$;Ok*{fb;hUN-o_~a)nwX9R!4;pxUb5hAQbkzj#n3
z-Fu7TjRmfEQaZ@!-7C884@+lX1_f*zj%s`$4YAo%jURie_)p_WN%;>_9u5_8J}E5_
z6O*oxr0kQXYU?2T-KpqKh9!q?fr2_B%R=ZRIu3zH?`9PCZTm4gqN8|}KI%P0Q9+9p
zQIbX12$b2{lZ+GBSisDsxzct##RawZ*0O8Zn_W0p>GZpz$Ef65%e|_%Opvoze7~Tm
zN;Y^|$p-Hf02`;Aq#V>)b1Fe-DQD_5)+#`ufyDql2Ze@r>0B64HgtzI{K4Rb%8m?T
z<^_Kp(W&w{0cA)|`H)gU?-z&rXk7pRa^oFYD;dlOTE+z?mS#l1L!FYFb+F*oqgCKA
zv>*&S>o*?+{=Apvh-Der3V9UP?gaMfOG^uBdkrCWi_SWoaS%@!MCb^fX(RIg#N|+7
z12~rfoI7JPw2mwIam1a*<vao>fN=zHK9n;9D3=VNTq=Naus8!zitgl-w8S`v)hyat
z^*<~q>`V)KGp7tg<??c~rHp0;sIAj_Siv<IXuy8CwP7`2U){PYuVJf;WMD3>dXc2{
z_%89GGJ^#rMV*%S(gu$2^n80kzOjFX&JfK=+b7vj(gm<g)Sr!2YYY@5E+$eJ&h-U&
z2+eSrG^(@@K_o$oVnL<63KbL9MGV-yCRPQeZ5|z9HR!^9FkUjZ4Hz3?wH-z>P;5#3
z<Z6m=Ma0<&iZj69H0<gU)kqAevXB<Sa-#Dwtt3`;600zog2_&z3KJelD-ZvqSO}+<
zM1aK@8EebAf%BM@AFS-u7E7I&LziDmIY*gF#IGwqbxLYk#DaxPY<VPE(W}qldfwHm
z6=8Q3b7`ZcbmC~6W5~$WI#A%`q>0t4W?ara=Z_fZ9PW=d1Ca7-3c8?D6m(5b)Ui@%
z3RH_O{R8cnJkuT}*fp7?dPNlN77zjlrRpmjS}^XhnssH|b78lI48B+>sIK<`rb&WR
z3Eaa%tzv@0zsaWEDSs4rywsfz0)n;M9BmO{huE$++&oNit9}%BPxQ%Kf>j#t$&TGz
zsrn1g3&{u{n`D_jLFbcn*d88Bvi}?dgt2gXxx{Ky0mf-BS#l98uQ}~L_)^*oCcb&C
zKRwcB!3gHeh6Cp;F6R;)&TeQ-&cU<!QJK*!VF;PBn3~*t5NKB4Hn>y?z>*qp9%G`z
z9T_AMo}`H^tdovlkeI7{2m8Q5vYNtR%M4&Pi+3fwqwLj}>^|~a$SMgc>Uy;fu-3Th
zNZnndixmNB2cZHR3T&8cG%}|P1f;9?7F`2h4>J>ru8X7-=iUh|BjPH9^=S$$2r#2R
z7pG8QqdcBYPa_qf&*MU$M}<B#H%_9)8tSWW9)gc7WkzOt<^T)Fpb+Q-?ebelf$fw>
zkDNyj2%5B7wD@@>ub{y-`d*?VL>Q?*rCnr6<7GU$nl)T~3O$F_=L{a*1l=QpzdX7J
zr0$#IRUegv+T)?_BQqoe-E_5Tb?VjXu7x4n{BahBJvd{)$llD&<ff%Bj2DL|5iuqX
z#d;Omj0uZ&s?=uWz1=e@2uN;~(y(iWMuG)5yR@<1y^0M#cK^BvuP3olqzuz!Wwc0D
zg))kuxniv>`XHK!lsgw!A@~pqsaPwwnMzp}q$HJ(IF03vmWWd(u}sV4GEm0~MCUT%
zcbnY?%#|DEiw%g}Mj0In+e_Lj@^QfNL54W2i5c~2s-tu0ha6An{1WM>BOC~(6jX4G
zNHh?I-1jUl(T!m>75tW_@eo6@bwteL(Q9#s8N6lO0LByXWGe9l1SHyHa3>B?H`lRO
zpz{J~sH#pWT0??cU#1I-7XCF<bh4}KEIOq$5#+qypA3s1Rl1NR1qJ6yMvN`(mtj{D
zMGGW0iY^kJtc8k^0n{jp!Q8-?D7Zj$4jxtFL-c)2z1oSF438nQkk9(v)@LClqrWFC
z|0vU7;;gHX{}MdHG(IM!|ER(ximmprM}a#Ot|fY4wNPtdV$R(FFJg~@mBhY<6k?=+
zILK1LRH3Fx-7vWK0L&&0W|AFZCh2f-iv5KB;2bXD05T2Ea8XGuKZRBGW$oJ$H4Ig<
zhV$Jo`27wz;CNW^5tcCy5h%+yjW)mL26kGZw*)G8Sp;6l*`H;;>}8@=cJtWTgMD@l
ztI-0Ue}Lil(~;>^Af1&?F%yvrajvM$o219OIVOjVUNAYfH<;8=4<)DkORQwTDZwib
z>$9BE==}q?sPfJ%(*6uH+U;C{J@)7PewHy!Eo0<Txtz`LI<DZyCOwVoZf6UrbFF3_
z-0`HaCV2=IxWfJ5G;A<QhpeqZ6?fUeE<>f={k|&Dzkg34DOoV~xw9-B=@`=)`*Q+!
z;tHCfbEV3pU=Rs*+^Thb7V^vZ%>IjXh8l=u{0hmVSO_sre@fNhPlf%dE-HQR_W0iK
zgoPHqm-7l8WqPa}S%y&V>+r;w-WP>!{1sviaBB@%1Bi=8!C($DkpXK+&VXfrEu;qN
zd2n&y4czx9xFrcY&rBQIQ<#=<TB5!Nn^~sLS>HU}LySS@MID%BaVzV8JIKyJf2<hH
zAcsGEImfL?K<sKOl3=<Qo)uh&)Dc&hFQmukQ3DzEqUc+gP;6Le$6-uM%ARUuX>M|5
zH;LV=n6=AT7ltitdLlBoMX-dB&t%7}q}$6h>BJ^LuOn5x=ZQ9O_~i|va#%YyD;Wv@
zfVDZ?vB=+lk8QLc9-qHst`0a|!fMexH|=N-LPz1X+yDQ}{=1|G2QrxbQLom!=k)e3
z;7dS<S9p6(R&Rd}(e9YT+rP-Ke~HdcO#M5o%vh#Azf;vv_`kry2kU;r*!T7rJ4eHp
zSc(AmCc)VE$|$MhUGO9SJdG>9nl~W|38f5DsF%thf6sWqL%k_My;JVgGtjY|_xHEL
zkQo^#>I78$@g8vKSv*8Jv?w>AfF^+IFN;tp<9DWBZ*-{#!Y1=e9m{!V@P=<|7cE$?
z^}NlUEm7W9MQl5<c(g%=0(bb;5_H=sm9Sev9NAG`b=FadO90Jo$yfS@VNl(YRoZ`z
z&KKzXbvk3j#lIubMO>srT+9#AW(T=JiU}howZ`n|j!Zy@G9}FA21g&E?-|Z8%JXR`
zz&W^N^+B{sl;>mR@DXx-?C3ita1jtu_M|Nl^l}!y1*izWiB&$dz2}1>(DMN%dcLd$
zg_XqP3Ly3ZViv&fu{{(|FlCk}D5n5EGmsSj4Qokh>`aqq_O2Tytg;y9#qq)l+(|c>
zge;=%mrg6LttkmB#KN#|ZZHX%Br{;7Q4oIyyXfvyCeB+^$_8w!i0|@{V3`#i^E#Y;
zAV$nn7FHLrUk_glisCPde+nsPWnW&BYnI;n;WR?0t9ttfD8FV+fo3r9@ZSX#D+LtW
z6Hsi%DtdI<YfXE;S*zsv=146?_g-NetCgS~EYUCl!CKeWywrQ%uQ$q2!b%(I6Ajxl
zCJZ|ZN8;@r6E-_0Y!0=S+CoZWrBvkcjHgrBvt<7YI+uIk76Vi*x%i__ExE*=g7r4B
zjIo0KBkD1Ds{nqw-M~gMr-!KoC8=w|JfeYz5bLwY$Lx|S6PKFZY8}?#cC)_w1)znJ
z0u;gl1XT#`e+<bBLvYhd(e1d?E42+ABvNZvRmj8Eu+_?8ek6cSW*>wi_FtwW^9=%#
zg?&3>5D_|Va<faz<K$I!>&q==snC?Ap0!O^>sQQ8@Ue>7hwJLrCa%7$$haxk!HD_t
zRqx*S?wht3?%n(9o~^Z7RfU*_co5;t56FXvpA?+I;tSx6{26eu7*oLHX2$!&-VDH;
z40dc_p9X!ruxA6)4}OD~;_5THeBrq=E!sEC-$HEUOC0Lm0%Us)<)T{N0*1u}-Jq3N
z%i_=q<EQnl<W>r6{~s5wr(O6cr?|3_pP1F>em3k0bl5j=V+A(d!@j}ve9#DazC1Xe
zdJ8G(`LI<1;}t8}_<BnG1@VL9tZ1+@NWm>_E@28j1uh6ZiItg`Y50XluuFFu8oTMi
z9&(|<9xF2kjrANhV?bL{!kd!3DaPMi{wl^Hrt3HZ!o*JE6EenOTY)sD=V(g-%Z$L<
zf_x&&-Nb`Z>>aEq+7HT?(__9UMZyphPvEnI0(fVJ7Cyo|vx7_(_9!T2M!3U$C>6D1
zd8Og8x6wMU8yDd4j*j=(P{($D8DQa<mEjsr?K#wxI3Kbr<KSGFiHt!R>{l6`lYvS~
zmi#uodtWV>p66?Uguh1LuhaPrI=@NhOLYDwov*;Dn0BMSDQ@5Xs|+0)Qq&E+6$08}
z$)H8W+}xCbQV?A#*Fy0pLyJQQSVNzOdH10dHu!6YIAAb%WWc<6c)*OnY7mh4L!8%R
z1(HJGdt?V3gqYRjP|hbrG>nIyccpSIuzE{4qoLN~VMP6J2?Ttqw${M$B%)Q7z2WC;
ztu=tVXK`A@`*Hk|^YF=Ii%*r8u~9@VJYqhFQzXzRJiSMfIDZMJoYY%2Cpg4}%bB!Z
zu%VOKa2n3B@`t!^$<5xtpg4|#o6etj-_rZe@LZ?GGFEF^tp-knkOfa{!IX5*Se7~u
z>iNz_d*w=}Q+~d~qoB^<$eEUWDADd(ZQEh*!G+$_pNkXe4FREcB)`&7eo+ic^=J~8
zV9+n3o5V1vw)t6)8(*XDK`AVuD**Np)JgnFA2jZsFC%D*VOhbS^yLJ9(nrJzB`t7b
zzLx|~!V}Txs<E0D%l{-+ih?kN6c9^=U@d8t+KQ<l1h`c~h}pbUxGfI|7N<!d7VX}G
z)U7P+$EMq}BjGuQ_j2HzUTi_9`BiMX19t`%1?uwOay@Gb*fo5cj=V+QgJSG0@Tz<}
z8TkTuRl=qtVZg2<@nB(x+>=!U$PNfQhU~z*zlJ8e4Yn;ioc)QEf_bs0z_LcTPV?7s
z63eR<WD#ZNSTPk{x-fBJ-68ObgC+pKMy>6dSS4u<qb)Ldpf~cO$Sf+*=_niDuk!8R
zqVrpHew)tk(D~bRgsJ}>_$q1UQ{C)(o~yp6-&F^+T(yT8IkC|OhX%QvkBu@u!trqq
zcCq3QY<nK+-$NSye9WUm6)cm;kjI5qUokZ(d3ZR7jGC-?Ca|(pB9d&3u=zqTeD!{L
zX!wpwjMqs`!A{j=#-gT3ivHQiEl6h@FDn-DMYuf2H4Ip+0FqfrEW>e;P8lC_kxm&O
zrz|f2PP+H7PG=SDf7+Y$ha|mgZu>U^@@f6I+%l^-g!%qPBzqC~JJ$4Alozu{P+4K%
zaz2O-#3dsj;?o`vKKXLudO{E}@bh(GVsKP!R`z@#t>?qeEMQ&Cwg>4IXnA?Y4^T6Z
zHubiH)UpnKgM}u>!WlYA+6#->3L6VXEvsp<&&_RqnrHr8_~h0EVly0?p}~Y4r-1c*
z9*+gYnX)oOvFeSat776T9WN&Mdn~5BiNrL5nBcFmnDPc|=GUG@y;&o0SS)s(Qcqk}
z&yqjOJMv~6vqcm<FoAfvEM^MV<asshblIidGfWSBiQQh%8PTYsJ`)=qPH<&04f}pk
z0I1QTmN&w7phP5_1P?uio9r1x&`;4XBc7woJ^|<Jes_Vk^<B5_t-<ta+%j8?Tjvzo
z4jDE1x#Gs}rtc^nu|a%-KH<yxv3&VF-@icT3Y|?j(A3!-u8}t^G3gR>JM!g9p3~Bc
z8$AgQ4|wq_E_Ego{xa+QAUw250E+flFg5N)VC5Wa1aU}!DMt(FDYD%Mc1f9l3;V-K
zxNspHPA~&SxBjsJ8Zq2J=f$!L22)C)4Si`*6wyccfkA`S){G3MECy3H7))t@Ffo=l
zm~xw+!4n3NKXmvm7&_>(oIDP4h@HaF!3l310BS%;On>O0I}gQ-3>}H-4;@VMhGP0d
zr;E;?qfv@l#)%Dx^c@a}*R`9802~tk@xEq%9i2o8{c)zLoSJ=yov92DI}`L4vCfoN
zUCEG6bo|zx=oX^&jx9j{9*Xs=ETEcyWem)}?zI^@UZ!$TjL1|T)23u9U^Un@4PVIA
za&Go<sN7+=7CD%S&?5#vc4=DjUwl4<IZ7!=_6?DcKa6BG8D<rghLnOPE;W4N=|SSr
zhJ^<<qUwgyV1Tob7o$@Wgt`+j@2TVX-!?9PMOl!RtQ`&^Ei$4XMM|<KEJ?s<y(xHV
z$oOe^Q9n#M<n4bS$@73iS%IKREoaya8{h7a;7!=>w{7`b#CaRggWwppXBz8V3GFY+
z%=K|hliQsu<z}zEO7o{yr?cjiYdx80FE?!jp|15ciTwQ=Km6MEAAj?=9u4-Iqxj)^
z5xKWoFY<a6xBfP|4u~&+{vYD~FyiLI_Wz2|c;9e2$Mi2YJDZ=Evjm5hubA%P=+v8u
zmjQUx6F4pa63<e>0fAz(OH1Atwd-2d;IV-S=hiCP;`-s{bt3UwTHhW<ej?eiCj-X>
zB45h)HOV_XdPGTh-Igj}!Y81Ci238nPkP|A<DTW2p$EeLCq6A8>>nJV1qQ%4$vz%~
zybXDXN$pXT6jBW()xwf`Mz#_T`7_4I)(A^4#!;~<7;1VZ?fM7ED1upj7LRNje}(w9
zs$uzNx-iG$oY6Hv7qR=1;p$*bu(|G3-ARmHJSdyIs=vw?j9#8C+fwAj$d=%+bVA<(
z68{s@mrpb@<9Q47T7PC7odcXe$j$34E>ZA_rPiXLgd-tJz|sy3<)G3PYb_Yd8Dd9n
z4q0Ep_<?HBxaw2c&qZwJ{xPB`%#|98xdbJ?%t#?)kt&j98Kgby4{-;bV<tI6e6)Y0
zk^v73s+z_(fwt%Uc#x{%KxlxKlVMfGf|X&)f>l-wQGIArfX1<_5ny7t+L(fRm6WRe
z6Qr%G`vLf3cDq9oUn=?M8GWdx6zqFgQ?@DvJ1tjXx5~O69;xdX$!-4weD``S;eP{Z
z{dd1>!E{xJ>1sMxv>9H<75q@*#A#f2vxHyzAda3q2WD>7Z0j6OM{n4V@17&)n@%-2
z>C9)%-$apyDNNQ}CWTQiP`H0!G}|4Rb0C+08XsavB<wQIAqB{lvpCRmV7v&{%R`sj
z^YVs=YC(2H-yj^?Efo(Iss4e?@*SRG?kKD}#`{QJF=nJHO1QKfi@EBZrXe*1A|RLX
zxS6Z157(P>uKWFZgAE@UQ|}!Zu|F{+<QQKt8k)SemF2e(Yh1A&2#bY5BaJ4ywcls_
z--YAmxVyS4iZA;gF(4}e;HYCbX8t*z?bgKNCqUwGejHc#GC##iQrdof9(M6Xy+1pi
zAORa8w+y<#B|Hclc~aQOd%;H7Ar^ggF=r~|nwZ*SyTAc%3DDb6+D5~8R53OXC^aXI
z;+aB)({?dh6SFcOa%2=)gpIJB?LVaRM|8+*!t)Cjc4n<^k>^Ou8if@({Z3IR;Iqd_
zoEfC4xK9fFXBM;50@inMNcq7rL>lF|cSI`eXa=b;NsN%n54$T;36+Fiw;)xVLA#}m
z_G=M~tol7MVu3_4V%;}}SYudrWVgW`&$s>=3iBowCIb)FyNuifwiqz9jw|>f#$EI2
zB5U<jv;9<U^LDwkbmG@y&d;zK9&_F|oY(*Rtl8d)3qI@Dn;3bd-v5_a^YdY$Mp(0J
ztk*U@P7Gfa&&{^~4E8&MW$kaH`;BABA7L&d40&c85yrFQ9ugLI9PZyDGks({J625C
zW6HBprtH5N%Wsx<$(sKaYxytf+=Ac4t$OW$K87+xy&orK;@tQkmS;R}ynlp3ufdCV
zODA&Td7d#5a%7KA@i%uxCn1p#>lSp1^JV*AjG+@5@#Pqugb_#Rba;eLZx9>)4vO+&
z7A3}p8Cu5`{1D@=+3>_;oh$8gwe>D=@@~(s3HBnSd)OMk+{MAWYmG)1M{2cvwb6G`
zH2dGs$p(&K1LEdutQs*;g4+;gA|)8Zms!b}LhiP0N?&GnA_?^VR5Y`w#*D=z-Z+cP
z`2#q@2mY?WvajMlJb~H-U`#u}eF6Ym=)XGl2VgHEeK^ZcUo-6Idi-xvFuQvZTa7JX
z5cu&{o?!jY-J$Iw;-nCsVkkpkHwAo;XB~6K)3Douy%E;<sXB-K6;}UKFJT>OJA<8P
znJO%zm?C&i2D{L>P9v5{2+KLWrxTZZ*iYc^i_9#opc?r7=skkG`$ZniY?o#4EE=u+
zVE@ev{{L!YHD732H5z`%|3c!mYx%(+YYY8(Y&U5;>>@BbQSb3zLp5+3$w8#PFySU`
z*ne1+tn3lsD_feM(+sLBY-OPSemtsatqevPtn4A!$jWX$ZbiG2@eAvklI{^kJWpq+
zTan4#U^^4>E6B<JlBj_Fyf{0OcnUUoIdnJtr$l?BePEQ+jdr-(U4Z_zpu=hXd!m_*
zVHf^E39^fIjWui0S*OFXGJF(PSa0#knEKa<_agK3*hLca>v@}@bzH%Z6+4aVZf4gL
z&orPObsFcfAME_Dr`1YZ<0(3&;lpYKT;WR*s?9{Qc|&-@SAxvI6QY?#c>?)Cpz(PQ
z59A3LYYF>s?93`y1<K-3@!7=Y3E>zIiav<PXw#ycb9?X)j$>~*33vJ8LF$7rQ^%fe
z8B2ze^}$fN#WZZkVes3m8pj@n0XsYf0kuCR?pfX&t?C=m^}i9&6<t4?*&EdL|1GNd
zo6OVe`X;)5-ezc>6!3qDp}PKq@PrEmp3%k53q;fB8(@3jXy^SsA%WuJP8W*46`|1n
zk>uZy&i?I4mU~CDjCOevDlTnzVt#@=hlVk^@OL7a9T?3l=yZ3nHsm6kZ!#MWP;YIB
zVGCuowTSYC|M3b`fR3!%yU8~9p^JK>k`IdDyw$20y|-HKUi_b!-K)piP^DuhI_ye#
z(2L`yy#V`Hkzi#`MRI5%S!;!sOSBeL=RH>P&RPp*{Ow~G@qb;~q@_)mBPS2axizv|
zU-qP`U4Fm9w_l^fHBDLI60N9AxorYIVdK)FZB{WN_X7U!W=^TI6d&avkkgZ4e&qd?
lU{VXifS*k?N>7*GSz0K)9sWFgkC$?#qoqas-CsKNe*hi1`K15=

literal 0
HcmV?d00001

diff --git a/scripts/pymo/__pycache__/viz_tools.cpython-37.pyc b/scripts/pymo/__pycache__/viz_tools.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0c6933b43e7675a857cdf559eb745b38244dea12
GIT binary patch
literal 6993
zcmcIpOK%%lcCK457N4SIS(4jsH(QaMW@3wK8<R|?CAn>PyC3LQ9NTDj*m2k;-XcYb
zWKp+D5>+YYF>YXb7Di^W2#{nW0YQMtKbTnz23aLQRsoX5U1T$hL_rpVL6C)4`OYnp
zdboRL5lj(v>b~!-Tjx99IhS|y`HY6^-(UIu##BkueoC3qWg+o4?qCmsYn-*UDpOCr
zYN)5#v8pyp>QdDdzswD8e!;2^zrs`8K`PB(<!PQlD#Nex9M2<_<*)GxUPLO#C;8MD
zS~bs0e1^}Wq`+V2m-s74O>p)|E6*QeZf*T>U8g55deXw~;SLrcVkUB-cF2x2riI#;
z9%G@i8rNwl&kb4}hOyQobE(&DN&}*GooQNGmv+M!9XFH)@3qPXs@kE<g{|ghSZlin
z{%$B!fxG85TFtUvwehI6xUezh4>aN6@nTSE`*pW{Bd7~EtZ!C&TYkqoaHERwgO*oo
zdO^4=yxNA>t#5W*v0bUR{jSF=_u9Kb=!sUhS?M5Ex#!<_)bqUhX6?OUm-oD`hcR1z
zw}Qc2y)dZs4my5iuNBoo-){#?y#wjc>S|aQttlX(F_Sq=&?mkB1O1p5DRL0?6a-yG
z+*E4d4&KH##1_}K9dQ*q$6}ka=WLLQQ-?a&kJ!M9@tIM3GuF0@L-R=cuGTeU<GWg{
ztMdCOw+0T%^nug=*_YZo+SC6Wq+@H#jva2q=^5=3z7KAPn<qJLahufRxy7jFRJ%du
zY|BCRJ6eM&1#)N&ar%<RvzIW>Gd9Te@5edJxHibEQSzhN1_d?BNqzz=O2wJD(A48>
zJkiui5qaUc$+Vq+c%sFoco18MPMkiw8a^?5e&r}>=tnyGJwQJTnpIpcGnv9i2wIVM
z>|<Y}++tAsv<DTS@27IVKgu<4sq3e=CEJk9FI$rJB|DIo+wN_;lGUY6isFenzKyy_
z-%Z>Xa0hb`O)b`*76&Y5UuxU7Fk|+8eaC#1sHD<MsJ6beET+*IYb$K|t!@}VZE9S&
z`>KUVqwYm!rQHg`y5H%o{Thu<Q`uUyK+S4F*s5>iwV=RDD@x6SH6%tIq*LE)wYl)R
zk-mN7n7U#vy}jT3>%XGwhqudVK{y4<)F3kJa$XB|H+pS9gz9d4Z7=k@(%kkA0%`BL
z?OiVb;OcG&Xh<_@=61V&=!RaoD6<`4tyc?$Ti=#eL%1DJn%oUtX(w+eGiu>oR0L9Y
z`yx-%8<<||ydf>sN;*9kiw<l3aq2)8NAEJ;B-%TfQpIRFOdKS;%fu{d)fUR)`Mn~v
z1veoy05QihI;3twI*=JQ#R|w5Sy8`&9G>i|ZZaF?S9C$$#z>{_O*mFaeF$yfjO%T!
z1AGEC32K)@9hk7W0n|FNvt@FVu=QLYBlj*L^w53@xu2jsHApMuru#ov$o(&%FU^6I
zF^;mxJE!tF<kP3}dEm+cqH=MbXLy!S^&E7dFk9dgUl3-kI0MWUhnOw&|1d59v!4tm
z)Lawexr%D8ll-Kb>m)ygxzd=c2n^@qNyvOW1tfz&_ys;q^X~k!!t&>6nTfOfBG8*t
zGOuKYD28m#&gVny&aDfA85GyP)Axc~@06KR8c{&0)NgUrWO~Bi^6FzKp9OYx1njSY
zxtPF6f~;=YmU<t031!!#K+xw^2o)ER`v0L_%!~%Z`ju`f!%r9&WHHupkHCHHyu#k%
zUj=V*34I9J;uR#Mwb2IZ%DT9WoLcr6=21a3H{3UtudCnE@02YuNsXqcAB!Xlq*eFZ
zz9?s_)16*h(bsk>43?1b!&_Y3qUK~O#1#^+lDJCZH4-HfualUEkY?!h!}0||%r6#6
zTqE)8B;FuVCPBzg#{CUS{U(X)ByNycf~bz{KqM+uMt~Q~B;KUjA>yN#5kK|v22zh;
z4MGF93H60@s3)8k4V&rs<%|*Xi^erzKLhkrO->gy1X!E2C1Gu{`*e{yW^pT@=jTWa
z`8Q#0L$S6Qn?u&tV>oGG?d!yt1J?gZXo6$emc<Pq(R^kAe}6x)`hSah`!IDxto@e*
zn=Ho{;1+PVjdrOaVpAB+iEV{b>yidL13o^L2iuRvzN`&0{SV_5#=ASns(#sVzntoK
zlF!FFY=;x)faG+X2a=Tqi8F}}**ScoICkglCy57F0MDL)Q{ncQ_U8<A$XGtY9nc;s
zqI8HTUna;C$RvDt7BF2JV#tQCfvMoz)jIy(R}x`@lwUykFW8M^vNF<a3iqIFBp6yj
zi82C1nmncZC2DL5x7+lj6>Pda?-U#e1m!}a{z~&t^j)dCf`n3arPgD8ms!P_HRWfV
z)An;)AhV}lc?0tWgvJv-v-LTn$mSG_6%rQH#rtS+Oh%(VQd;^1#D+>~Kg14@i%BE~
zvNGYYD+m3EsLBF~X&^7<7Lv<OZamy~haA7>#=!a>>pP)Es!7h@GX@3wUnp#Dkm~<;
z2nF484%0_Tx%>mTqJuQKcTi^3D3@_!sLUAv2SlTk5=tK1u`#X9YEV`X$w~Pjmrw}k
z1LP8;9l5Xo#q_ksFNWE(CC?b2c@9#>JLUN75o22aVVuDlKZIjS^Cgbx>G=w3zHvFo
z(Avs_+{i)ADF=0qqM#TNl_IvE0Sp}XM?)}NS%)ot8=EWcKn#I_BHC+ZI*=xvQ$m{V
zZs^NY*MsL1gcQrTK@XHB^KcA3(ev9X(irujL5?Y40Q;q2x`UqERRaxe{y*LRRbcKr
zm~e!-c?xst-7aFAHWv8x1oaDGH}<rv=0kpWw^am1Bt)D#At;4FY1TV#?}UuSTUd%T
z{6<6EMJn0UcW6Ie!fxS&x2n0JZ+o*TT3p(#PO$0kOZHg2OM~2}G1Fdmw?lE=X+*zH
zHHNzp;5+qOfugx`v07+&u8JPvw_`g_5HF1;R;$3Su&)sR9tNqVl!seuB~*77|GxzC
zg|j<8z;FRs5ilBlQ^5KRGxa&3eh$8C!elypS)I*+!7`9VT|7biv1(YzP)Mra-$nxP
z)tYPo6EM*GRYdKLm{HU|FhVdE;yFDuC~A-OX)RU;Ay#~KNoyE005@P&A~T^%!9)O9
zVSwo8(27k0-OfV*q%Z`Cop^Rpb}_h7S`12WEC%IBk4w@{+<#@Y;gY3=?}s%C84&*a
zJ!!PPuC#gx%|m5$rS0~5UYAp#_9?7!NWq5@x*N$JE3ZLu$CyCKVMOm51vr0$*H%<S
z#*^H9gaNF?rkQY2^B<GyI=Zbi?8W3}81}wZsf4Z2?SAW<x4!v}I_)7{<j#;cr}pwk
zNPtV(wl01v)ShVrD}+(y#xwoTrv^6SSe$^w_7P%K+Kb=Q9%`S0^#`f$1N5@c>)WB)
z-7O1y?)0z54t8<|Cex17h$kP!&LL82M=8d|j&G$XT1RZ0R_7@hjAhcP3eU!7oWi%#
z8|DnSpXcJttj6<XP<eskgh4KJw$g|2#^c<qc0>m{6YJ90*xQVp5VmLuDarPui%Uxt
z1bW_v?{5c{aSc96ARR#wAVtzimwK>w9}%wA>T-`lGrzst=>}gl1X@PFyBJhd;Ml{F
zjwdR~V`($&v~N@JSlbot)y3c%;?B0Wy4bi@k5+FkFJJQo_rz*HI%r)>h%;<;s6NWA
zv{XSU5cSn)zO>)s;pXc6o6F0K^QBFXPLEc<vAn#tQc2paMdcN@v?;s>rYVK4XnNu5
zd<_nDcYA(q-P<oc!JGK|D;0OGoR#UjpS=G>v{VS}L-VAyPdRB)<efOfEhvDrg-0HV
zv|%3bo-*~pgM)Ar@pOu}0(wblzW?~+kIH#zsn?f!ry(;ZuL51BH<r$Z8+fz{Qs3CG
zT0Sf;3Q6|da8p?gsn_e$peA<mO0<~n_Ia=2Vt1oP6T?%YIK3hK4*L8Z78{V$LJ+WF
zOJ-m_C??K87GPOy#KlFF<Jjtt(dzhrz*}d6;IZQGp$5hS$5UMYZB0E5dXisps4HoN
zFa<ccg2Sp@;5Aih(Qyzw4SNWH_q+dsl)bHgkBOPkhKi@Q9DpGWFr*b2G86^_3_MOB
zBH&eE0RO6%89)JHFoJCI8no@S_(yVG;ce)L2!W5_ozUSFc_4?DI_E?fN@YiMEIA2q
z<N=>4_y#HX3m4)nR#u1$NKeQ4ar$D2;2M2Kbu-ADr}DGN8%aI^V^|o!>)esfFTorX
z;a5x?PQ(RT|0!ee3i&;;NoI9W<d@?j)^!Ezn&hv>MU+j2#jVN1sdzG;8iMxfdUR#a
z6(vH<-Q7mR6Q$MC=L-w(+<AOwVd3+)qB*MVc%e%zP`9w~@w-&@R<s72DZ)2SdR`y5
zyN+h}stXJBe01;5NAD)d`w#D+^9LUzTYf9Ld>*~Y{0hN>Lt{jPL=ysLL2Ocn5Hcnr
z9T_sDDlVW%S;Ykf5RrBWLwYof4$ABWyAF@XBof6y)KZ#&Ugw;mtC0F%Q9JThUO%Jw
zU*)Vw1K54RJ`;b6Zf6!|&^UC{bU4>j%Zk~I_6bh2cXV|Gu1Dr)i^0=%wT{usSo_?v
zUZhu=I5h6}#M-~nfb`m?7L~qkaihjB*waY=>}lkTz9yxTj~Cg=_leTu?-S)07EXPi
z=)&o*L*IC5D-a&0j;=2Tw||*29b4V#%E+oJ3$>A0-}&YFUx}5yAa(GHNd}J0rBL+m
z2oJFl6*)EXJH1*>{25+EbV!s)kXi|f4TTQ@*B}TFdY()P4|cEaiDy)`O@jO#(Ic@#
zLO|fmf<CkeA*4xA5xbP%gQz-ncp$>9MPi?_dD_VuR1f2Z9{x>fiJDrPN$$8Z$|$5Z
zWcn>wV=t&WG$s`cySqWqYU1Ap81?&j9q~srTTFtyyK~wj^Jnzr$7m5yh?@MN;IL@}
z*4lL7?`2@CmF?DxIOUk8^Glt4L^|V`GE8%ZP3t)0BzyiB=wD`!@#o9>{dGYH7^joQ
z`ntHST8IzmVTcVp5^uYnv^`bwCn#CRDKPxEE_ZQ;*5lHGH;unR^lnd1;(r`EPHML{
z@Xv@kJGPT&;-P;@LuW|bBta)FDr8kYtMbMa`=}+K+L{xu&ROaBe7EhbfrJvYk}rqX
T)H8NwG4p2Ta^|X&&CLBD;!1ok

literal 0
HcmV?d00001

diff --git a/scripts/pymo/__pycache__/writers.cpython-37.pyc b/scripts/pymo/__pycache__/writers.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0d38b69b5eb765e6ad377d59b7dae3a7e182f063
GIT binary patch
literal 2634
zcmbVO%~KmU6qmHS_IhoI;}SlSw8PMGJMJ{NAJa>Rp(K=qWKxp}OlX<O>@3zAY}UJr
zw6w5UpWqKj{sVLDwTJ#OI`q;Lx1M_7(!M8SzDh5VN6*sJ)6>)Y_-P-Fj5q|^A9vqv
zF2LwdoD7=*g9Yev1_&pd24sWkf3{&fA)ImJh;ZYOZJ3;{lA3t}a-`8jI&9}UJ^_xe
z_ir#xIXfa71}|`vTSpZ3=Xf4C7Pmo<qY95+Ew77q;)^Cd(}e%n7okfN2qz#)8@~ck
zru2iBz_T<teNzvjrWZVrP2nZY?Rwmay8eNe)<q=SzPshiWKa0+rXM!9yPnvoH-jkj
zd3`b1lW=G|+^TnBR9}o9tj50I+;*RwZF%B{KIm*mVI7pW<3!fugKkv+MO!Op;{#>8
zZaZuzu4|#G3<#lV<@`po!{w8Ipf&L-*ibkJUEl9_7-XyuE<dJ}Y>|w(G$VbM(TsKs
z&N6mv(t(755?a=THM3)JHbusX=7QvoDK_=0mE?FXGwzW?I#><M13v7^4>*b4FJz<x
zXLcMsk>|O61ZQ_>znGMc>G!a=XX;&r{s?z?5mrs)9&0;wL`sd+S$?|o^5M(H<)0jZ
zAxrZ=tgJm<X*g-|vGBUS{C2v^o%EKTOs}=OJ}_F&7KlRo>$~%qqwml!tI*{*5cI_B
z(H<mWpJZ_6JGN_L7M#*)%z{qT4r@%Y*)Nm{$<w{a2!U!&ow=YblV;S7=kK9P)Sn~i
z#}~2DVVs@Q*(2v+FqBv1idC3u5<6Nov?&A6zJes(g#-tySZY+I)6hoLn7}$yG`Mb4
zVQvdjYCjh9_fa33WRi5||4kZ-*G844%~f_c!i1ErD!MTwZ}JTMkf?%)qQnis;VSpD
z7wq{`S#sNpePy+R$OC~EMgy|hS1gQG-jkjX-hrZPD(A`M03;2W2vGo=YL>uERrE(?
zMXh$=iwT@sK|A#2wL6J2EZNYtcZlJz9IY4*9cMP3WM!jF$7y=&+6|i+A#9`2@kCt+
zpMgIhK_S7XpctT&9VTWnvkwT8v9m;8NT4vzWQ-|DM{YveztB<^CjA)fZw;j@nx7x=
z&k6<g+5h1bki(q`0{e&hi(rE}b#?Aso0q0Y%RJLO+&3~sde*s?QJCfUEnbEK%IUJo
z^V^qZHryzGNP*gibb|B>ywY<rK+7HTEfv4*KTnLo+94F>>$zSrvoo-)SAt?S`%vor
zJe+0o4>FdO@a9vbV-JcnbNJW<$x5Krc}sVv*UA11tqso8+7^06ekZd~-}w7z4A;p`
z@p#L`nD+emC9f7Tr)3O0wg36KcOZsvtGm4sfV=`g-pC*dnTenWHvhd2umcgBxEwJJ
zK9dcVi#_3oNov1bSy`Kg0#tT$yB%;~4#22bmCsh5HuTylEu=7C3i-4Iz;IHtD&IKi
zy{cS!{CIT<WWJvLrgfP*Cx~b&AGKQ2Pvi;KD^!HwzQBXLF!Td?1uC#?G=NRQKm%AA
z0W70CIRTJ8gfL}`DA5ngRlsOoFRl`(`4ajNy=oVQbgBAp!m#%<D~2Z%;pMVfq6P-s
zWY||1hKUgvr&d+ERVfe+QU?ZScTCGwxw!nW(O7!E3XcJV30>cF;O)2Njvx359EdQY
zjKB|-CA@IU7q^j-^WxYKxj=V{DI{Y^DnM$Z8l=RBFjf}628l8fZ&Q4P48X(0)sYgP
z;@p6_GT?g33Eg4nHgfRT1L!H%Y(o!kyeXcAD&-sy!YULIZIV^sSB7z!PSR0hl)?|@
zCUt0q+H?kJg_iXHB)iYX=v|iHxiNT+8WnlaT+89d1R}WWD#vxZ5#I}NTyovrJueun
z2>fi&&eBe}6c+tKgt#gO&k9(KvNp91XI%4Yma@XVZhWBKiNzCgPbwC3y?m1{WzP-l
Nn8!-cpJJep{R{q$R8ar`

literal 0
HcmV?d00001

diff --git a/scripts/pymo/data.py b/scripts/pymo/data.py
new file mode 100644
index 0000000..448fbb0
--- /dev/null
+++ b/scripts/pymo/data.py
@@ -0,0 +1,53 @@
+import numpy as np
+
+class Joint():
+    def __init__(self, name, parent=None, children=None):
+        self.name = name
+        self.parent = parent
+        self.children = children
+
+class MocapData():
+    def __init__(self):
+        self.skeleton = {}
+        self.values = None
+        self.channel_names = []
+        self.framerate = 0.0
+        self.root_name = ''
+    
+    def traverse(self, j=None):
+        stack = [self.root_name]
+        while stack:
+            joint = stack.pop()
+            yield joint
+            for c in self.skeleton[joint]['children']:
+                stack.append(c)
+
+    def clone(self):
+        import copy
+        new_data = MocapData()
+        new_data.skeleton = copy.deepcopy(self.skeleton)
+        new_data.values = copy.deepcopy(self.values)
+        new_data.channel_names = copy.deepcopy(self.channel_names)
+        new_data.root_name = copy.deepcopy(self.root_name)
+        new_data.framerate = copy.deepcopy(self.framerate)
+        return new_data
+
+    def get_all_channels(self):
+        '''Returns all of the channels parsed from the file as a 2D numpy array'''
+
+        frames = [f[1] for f in self.values]
+        return np.asarray([[channel[2] for channel in frame] for frame in frames])
+
+    def get_skeleton_tree(self):
+        tree = []
+        root_key =  [j for j in self.skeleton if self.skeleton[j]['parent']==None][0]
+        
+        root_joint = Joint(root_key)
+    
+    def get_empty_channels(self):
+        #TODO
+        pass
+
+    def get_constant_channels(self):
+        #TODO
+        pass
diff --git a/scripts/pymo/features.py b/scripts/pymo/features.py
new file mode 100644
index 0000000..fec29ed
--- /dev/null
+++ b/scripts/pymo/features.py
@@ -0,0 +1,43 @@
+'''
+A set of mocap feature extraction functions
+
+Created by Omid Alemi | Nov 17 2017
+
+'''
+import numpy as np
+import pandas as pd
+import peakutils
+import matplotlib.pyplot as plt
+
+def get_foot_contact_idxs(signal, t=0.02, min_dist=120):
+    up_idxs = peakutils.indexes(signal, thres=t/max(signal), min_dist=min_dist)
+    down_idxs = peakutils.indexes(-signal, thres=t/min(signal), min_dist=min_dist)
+
+    return [up_idxs, down_idxs]
+
+
+def create_foot_contact_signal(mocap_track, col_name, start=1, t=0.02, min_dist=120):
+    signal = mocap_track.values[col_name].values
+    idxs = get_foot_contact_idxs(signal, t, min_dist)
+
+    step_signal = []
+
+    c = start
+    for f in range(len(signal)):    
+        if f in idxs[1]:
+            c = 0
+        elif f in idxs[0]:
+            c = 1
+                                            
+        step_signal.append(c)
+    
+    return step_signal
+
+def plot_foot_up_down(mocap_track, col_name, t=0.02, min_dist=120):
+    
+    signal = mocap_track.values[col_name].values
+    idxs = get_foot_contact_idxs(signal, t, min_dist)
+
+    plt.plot(mocap_track.values.index, signal)
+    plt.plot(mocap_track.values.index[idxs[0]], signal[idxs[0]], 'ro')
+    plt.plot(mocap_track.values.index[idxs[1]], signal[idxs[1]], 'go')
diff --git a/scripts/pymo/parsers.py b/scripts/pymo/parsers.py
new file mode 100644
index 0000000..b5fece8
--- /dev/null
+++ b/scripts/pymo/parsers.py
@@ -0,0 +1,260 @@
+'''
+BVH Parser Class
+
+By Omid Alemi
+Created: June 12, 2017
+
+Based on: https://gist.github.com/johnfredcee/2007503
+
+'''
+import re
+import numpy as np
+from pymo.data import Joint, MocapData
+
+class BVHScanner():
+    '''
+    A wrapper class for re.Scanner
+    '''
+    def __init__(self):
+
+        def identifier(scanner, token):
+            return 'IDENT', token
+
+        def operator(scanner, token):
+            return 'OPERATOR', token
+
+        def digit(scanner, token):
+            return 'DIGIT', token
+
+        def open_brace(scanner, token):
+            return 'OPEN_BRACE', token
+
+        def close_brace(scanner, token):
+            return 'CLOSE_BRACE', token
+
+        self.scanner = re.Scanner([
+            (r'[a-zA-Z_]\w*', identifier),
+            #(r'-*[0-9]+(\.[0-9]+)?', digit), # won't work for .34
+            #(r'[-+]?[0-9]*\.?[0-9]+', digit), # won't work for 4.56e-2
+            #(r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', digit),
+            (r'-*[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', digit),
+            (r'}', close_brace),
+            (r'}', close_brace),
+            (r'{', open_brace),
+            (r':', None),
+            (r'\s+', None)
+        ])
+
+    def scan(self, stuff):
+        return self.scanner.scan(stuff)
+
+
+
+class BVHParser():
+    '''
+    A class to parse a BVH file.
+    
+    Extracts the skeleton and channel values
+    '''
+    def __init__(self, filename=None):
+        self.reset()
+
+    def reset(self): 
+        self._skeleton = {}
+        self.bone_context = []
+        self._motion_channels = []
+        self._motions = []
+        self.current_token = 0
+        self.framerate = 0.0
+        self.root_name = ''
+
+        self.scanner = BVHScanner()
+        
+        self.data = MocapData()
+
+
+    def parse(self, filename, start=0, stop=-1):
+        self.reset()
+
+        with open(filename, 'r') as bvh_file:
+            raw_contents = bvh_file.read()
+        tokens, remainder = self.scanner.scan(raw_contents)
+        self._parse_hierarchy(tokens)
+        self.current_token = self.current_token + 1
+        self._parse_motion(tokens, start, stop)
+        
+        self.data.skeleton = self._skeleton
+        self.data.channel_names = self._motion_channels
+        self.data.values = self._to_DataFrame()
+        self.data.root_name = self.root_name
+        self.data.framerate = self.framerate
+
+        return self.data
+    
+    def _to_DataFrame(self):
+        '''Returns all of the channels parsed from the file as a pandas DataFrame'''
+
+        import pandas as pd
+        time_index = pd.to_timedelta([f[0] for f in self._motions], unit='s')
+        frames = [f[1] for f in self._motions]
+        channels = np.asarray([[channel[2] for channel in frame] for frame in frames])
+        column_names = ['%s_%s'%(c[0], c[1]) for c in self._motion_channels]
+
+        return pd.DataFrame(data=channels, index=time_index, columns=column_names)
+
+
+    def _new_bone(self, parent, name):
+        bone = {'parent': parent, 'channels': [], 'offsets': [], 'order': '','children': []}
+        return bone
+
+    def _push_bone_context(self,name):
+        self.bone_context.append(name)
+
+    def _get_bone_context(self):
+        return self.bone_context[len(self.bone_context)-1]
+
+    def _pop_bone_context(self):
+        self.bone_context = self.bone_context[:-1]
+        return self.bone_context[len(self.bone_context)-1]
+
+    def _read_offset(self, bvh, token_index):
+        if bvh[token_index] != ('IDENT', 'OFFSET'):
+            return None, None
+        token_index = token_index + 1
+        offsets = [0.0] * 3
+        for i in range(3):
+            offsets[i] = float(bvh[token_index][1])
+            token_index = token_index + 1
+        return offsets, token_index
+    
+    def _read_channels(self, bvh, token_index):
+        if bvh[token_index] != ('IDENT', 'CHANNELS'):
+            return None, None
+        token_index = token_index + 1
+        channel_count = int(bvh[token_index][1])
+        token_index = token_index + 1
+        channels = [""] * channel_count
+        order = ""
+        for i in range(channel_count):
+            channels[i] = bvh[token_index][1]
+            token_index = token_index + 1
+            if(channels[i] == "Xrotation" or channels[i]== "Yrotation" or channels[i]== "Zrotation"):
+                order += channels[i][0]
+            else :
+                order = ""
+        return channels, token_index, order
+
+    def _parse_joint(self, bvh, token_index):
+        end_site = False
+        joint_id = bvh[token_index][1]
+        token_index = token_index + 1
+        joint_name = bvh[token_index][1]
+        token_index = token_index + 1
+        
+        parent_name = self._get_bone_context()
+
+        if (joint_id == "End"):
+            joint_name = parent_name+ '_Nub'
+            end_site = True
+        joint = self._new_bone(parent_name, joint_name)
+        if bvh[token_index][0] != 'OPEN_BRACE':
+            print('Was expecting brance, got ', bvh[token_index])
+            return None
+        token_index = token_index + 1
+        offsets, token_index = self._read_offset(bvh, token_index)
+        joint['offsets'] = offsets
+        if not end_site:
+            channels, token_index, order = self._read_channels(bvh, token_index)
+            joint['channels'] = channels
+            joint['order'] = order
+            for channel in channels:
+                self._motion_channels.append((joint_name, channel))
+
+        self._skeleton[joint_name] = joint
+        self._skeleton[parent_name]['children'].append(joint_name)
+
+        while (bvh[token_index][0] == 'IDENT' and bvh[token_index][1] == 'JOINT') or  (bvh[token_index][0] == 'IDENT' and bvh[token_index][1] == 'End'):
+            self._push_bone_context(joint_name)
+            token_index = self._parse_joint(bvh, token_index)
+            self._pop_bone_context()
+
+        if bvh[token_index][0] == 'CLOSE_BRACE':
+            return token_index + 1
+
+        print('Unexpected token ', bvh[token_index])
+
+    def _parse_hierarchy(self, bvh):
+        self.current_token = 0
+        if bvh[self.current_token] != ('IDENT', 'HIERARCHY'):
+            return None
+        self.current_token = self.current_token + 1
+        if bvh[self.current_token] != ('IDENT', 'ROOT'):
+            return None
+        self.current_token = self.current_token + 1
+        if bvh[self.current_token][0] != 'IDENT':
+            return None
+
+        root_name = bvh[self.current_token][1]
+        root_bone = self._new_bone(None, root_name)
+        self.current_token = self.current_token + 2 #skipping open brace
+        offsets, self.current_token = self._read_offset(bvh, self.current_token)
+        channels, self.current_token, order = self._read_channels(bvh, self.current_token)
+        root_bone['offsets'] = offsets
+        root_bone['channels'] = channels
+        root_bone['order'] = order
+        self._skeleton[root_name] = root_bone
+        self._push_bone_context(root_name)
+        
+        for channel in channels:
+            self._motion_channels.append((root_name, channel))
+
+        while bvh[self.current_token][1] == 'JOINT':
+            self.current_token = self._parse_joint(bvh, self.current_token)
+        
+        self.root_name = root_name
+
+    def _parse_motion(self, bvh, start, stop):
+        if bvh[self.current_token][0] != 'IDENT':
+            print('Unexpected text')
+            return None
+        if bvh[self.current_token][1] != 'MOTION':
+            print('No motion section')
+            return None
+        self.current_token = self.current_token + 1
+        if bvh[self.current_token][1] != 'Frames':
+            return None
+        self.current_token = self.current_token + 1
+        frame_count = int(bvh[self.current_token][1])
+        
+        if stop<0 or stop>frame_count:
+            stop = frame_count
+            
+        assert(start>=0)
+        assert(start<stop)
+        
+        self.current_token = self.current_token + 1
+        if bvh[self.current_token][1] != 'Frame':
+            return None
+        self.current_token = self.current_token + 1
+        if bvh[self.current_token][1] != 'Time':
+            return None
+        self.current_token = self.current_token + 1
+        frame_rate = float(bvh[self.current_token][1])
+
+        self.framerate = frame_rate
+       
+        self.current_token = self.current_token + 1
+       
+        frame_time = 0.0
+        self._motions = [()] * (stop-start)
+        idx=0
+        for i in range(stop):
+            channel_values = []
+            for channel in self._motion_channels:
+                channel_values.append((channel[0], channel[1], float(bvh[self.current_token][1])))
+                self.current_token = self.current_token + 1
+
+            if i>=start:
+                self._motions[idx] = (frame_time, channel_values)
+                frame_time = frame_time + frame_rate
+                idx+=1
diff --git a/scripts/pymo/preprocessing.py b/scripts/pymo/preprocessing.py
new file mode 100644
index 0000000..758f266
--- /dev/null
+++ b/scripts/pymo/preprocessing.py
@@ -0,0 +1,975 @@
+'''
+Preprocessing Tranformers Based on sci-kit's API
+
+By Omid Alemi
+Created on June 12, 2017
+
+Modified by Simon Alexanderson, 2020-06-24
+'''
+import copy
+import pandas as pd
+import numpy as np
+import transforms3d as t3d
+import scipy.ndimage.filters as filters
+from scipy.spatial.transform import Rotation as R
+
+from sklearn.base import BaseEstimator, TransformerMixin
+
+class MocapParameterizer(BaseEstimator, TransformerMixin):
+    def __init__(self, param_type = 'euler'):
+        '''
+        
+        param_type = {'euler', 'quat', 'expmap', 'position', 'expmap2pos'}
+        '''
+        self.param_type = param_type
+
+    def fit(self, X, y=None):
+        return self
+
+    def transform(self, X, y=None):
+        print("MocapParameterizer: " + self.param_type)
+        if self.param_type == 'euler':
+            return X
+        elif self.param_type == 'expmap':
+            return self._to_expmap(X)
+        elif self.param_type == 'quat':
+            return X
+        elif self.param_type == 'position':
+            return self._to_pos(X)
+        elif self.param_type == 'expmap2pos':
+            return self._expmap_to_pos(X)
+        else:
+            raise 'param types: euler, quat, expmap, position, expmap2pos'
+
+#        return X
+    
+    def inverse_transform(self, X, copy=None): 
+        if self.param_type == 'euler':
+            return X
+        elif self.param_type == 'expmap':
+            return self._expmap_to_euler(X)
+        elif self.param_type == 'quat':
+            raise 'quat2euler is not supported'
+        elif self.param_type == 'position':
+            raise 'positions 2 eulers is not supported'
+            return X
+        else:
+            raise 'param types: euler, quat, expmap, position'
+
+    def fix_rotvec(self, rots):
+        '''fix problems with discontinuous rotation vectors'''
+        new_rots = rots.copy()
+        
+        # Compute angles and alternative rotation angles
+        angs = np.linalg.norm(rots, axis=1)
+        alt_angs=2*np.pi-angs
+
+        #find discontinuities by checking if the alternative representation is closer
+        d_angs = np.diff(angs, axis=0)
+        d_angs2 = alt_angs[1:]-angs[:-1]        
+        swps = np.where(np.abs(d_angs2)<np.abs(d_angs))[0]
+
+        #reshape into intervals where we should flip rotation axis
+        isodd = swps.shape[0] % 2 == 1
+        if isodd:
+            swps=swps[:-1]
+        intv = 1+swps.reshape((swps.shape[0]//2, 2))
+        
+        #flip rotations in selected intervals
+        for ii in range(intv.shape[0]):
+            new_ax = -rots[intv[ii,0]:intv[ii,1],:]/np.tile(angs[intv[ii,0]:intv[ii,1], None], (1,3))
+            new_angs = alt_angs[intv[ii,0]:intv[ii,1]]
+            new_rots[intv[ii,0]:intv[ii,1],:] = new_ax*np.tile(new_angs[:, None], (1,3))
+
+        return new_rots
+        
+    def _to_pos(self, X):
+        '''Converts joints rotations in Euler angles to joint positions'''
+
+        Q = []
+        for track in X:
+            channels = []
+            titles = []
+            euler_df = track.values
+
+            # Create a new DataFrame to store the exponential map rep
+            pos_df = pd.DataFrame(index=euler_df.index)
+
+            # List the columns that contain rotation channels
+            rot_cols = [c for c in euler_df.columns if ('rotation' in c)]
+
+            # List the columns that contain position channels
+            pos_cols = [c for c in euler_df.columns if ('position' in c)]
+
+            # List the joints that are not end sites, i.e., have channels
+            joints = (joint for joint in track.skeleton)
+            
+            tree_data = {}
+
+            for joint in joints:
+                parent = track.skeleton[joint]['parent']
+                rot_order = track.skeleton[joint]['order']
+
+                # Get the rotation columns that belong to this joint
+                rc = euler_df[[c for c in rot_cols if joint in c]]
+
+                # Get the position columns that belong to this joint
+                pc = euler_df[[c for c in pos_cols if joint in c]]
+
+                # Make sure the columns are organized in xyz order
+                if rc.shape[1] < 3:
+                    euler_values = [[0,0,0] for f in rc.iterrows()]
+                    rot_order = "XYZ"
+                else:
+                    euler_values = [[f[1]['%s_%srotation'%(joint, rot_order[0])], 
+                                     f[1]['%s_%srotation'%(joint, rot_order[1])], 
+                                     f[1]['%s_%srotation'%(joint, rot_order[2])]] for f in rc.iterrows()]
+
+                if pc.shape[1] < 3:
+                    pos_values = [[0,0,0] for f in pc.iterrows()]
+                else:
+                    pos_values =[[f[1]['%s_Xposition'%joint], 
+                                  f[1]['%s_Yposition'%joint], 
+                                  f[1]['%s_Zposition'%joint]] for f in pc.iterrows()]
+                
+                # Convert the eulers to rotation matrices
+                rotmats = R.from_euler(rot_order, euler_values, degrees=True).inv()                  
+                tree_data[joint]=[
+                                    [], # to store the rotation matrix
+                                    []  # to store the calculated position
+                                 ] 
+
+                if track.root_name == joint:
+                    tree_data[joint][0] = rotmats
+                    tree_data[joint][1] = pos_values
+                else:
+                    # for every frame i, multiply this joint's rotmat to the rotmat of its parent
+                    tree_data[joint][0] = rotmats*tree_data[parent][0]
+
+                    # add the position channel to the offset and store it in k, for every frame i
+                    k = pos_values + np.asarray(track.skeleton[joint]['offsets'])
+
+                    # multiply k to the rotmat of the parent for every frame i
+                    q = tree_data[parent][0].inv().apply(k)
+
+                    # add q to the position of the parent, for every frame i
+                    tree_data[joint][1] = tree_data[parent][1] + q
+
+
+                # Create the corresponding columns in the new DataFrame
+                pos_df['%s_Xposition'%joint] = pd.Series(data=[e[0] for e in tree_data[joint][1]], index=pos_df.index)
+                pos_df['%s_Yposition'%joint] = pd.Series(data=[e[1] for e in tree_data[joint][1]], index=pos_df.index)
+                pos_df['%s_Zposition'%joint] = pd.Series(data=[e[2] for e in tree_data[joint][1]], index=pos_df.index)
+
+
+            new_track = track.clone()
+            new_track.values = pos_df
+            Q.append(new_track)
+        return Q
+
+    def _to_expmap(self, X):
+        '''Converts Euler angles to Exponential Maps'''
+
+        Q = []
+        for track in X:
+            channels = []
+            titles = []
+            euler_df = track.values
+
+            # Create a new DataFrame to store the exponential map rep
+            exp_df = euler_df.copy()
+
+            # List the columns that contain rotation channels
+            rots = [c for c in euler_df.columns if ('rotation' in c and 'Nub' not in c)]
+
+            # List the joints that are not end sites, i.e., have channels
+            joints = (joint for joint in track.skeleton if 'Nub' not in joint)
+
+            for joint in joints:
+                r = euler_df[[c for c in rots if joint in c]] # Get the columns that belong to this joint
+                rot_order = track.skeleton[joint]['order']
+                r1_col = '%s_%srotation'%(joint, rot_order[0])
+                r2_col = '%s_%srotation'%(joint, rot_order[1])
+                r3_col = '%s_%srotation'%(joint, rot_order[2])
+                
+                exp_df.drop([r1_col, r2_col, r3_col], axis=1, inplace=True)
+                euler = [[f[1][r1_col], f[1][r2_col], f[1][r3_col]] for f in r.iterrows()]
+                exps = np.array(self.fix_rotvec(R.from_euler(rot_order.lower(), euler, degrees=True).as_rotvec()))
+
+                # Create the corresponding columns in the new DataFrame    
+                exp_df.insert(loc=0, column='%s_gamma'%joint, value=pd.Series(data=[e[2] for e in exps], index=exp_df.index))
+                exp_df.insert(loc=0, column='%s_beta'%joint, value=pd.Series(data=[e[1] for e in exps], index=exp_df.index))
+                exp_df.insert(loc=0, column='%s_alpha'%joint, value=pd.Series(data=[e[0] for e in exps], index=exp_df.index))
+
+            #print(exp_df.columns)
+            new_track = track.clone()
+            new_track.values = exp_df
+            Q.append(new_track)
+
+        return Q
+
+    def _expmap_to_euler(self, X):
+        Q = []
+        for track in X:
+            channels = []
+            titles = []
+            exp_df = track.values
+
+            # Create a new DataFrame to store the exponential map rep
+            euler_df = exp_df.copy()
+            
+            # List the columns that contain rotation channels
+            exp_params = [c for c in exp_df.columns if ( any(p in c for p in ['alpha', 'beta','gamma']) and 'Nub' not in c)]
+
+            # List the joints that are not end sites, i.e., have channels
+            joints = (joint for joint in track.skeleton if 'Nub' not in joint)
+
+            for joint in joints:
+                r = exp_df[[c for c in exp_params if joint in c]] # Get the columns that belong to this joint
+                
+                euler_df.drop(['%s_alpha'%joint, '%s_beta'%joint, '%s_gamma'%joint], axis=1, inplace=True)
+                expmap = [[f[1]['%s_alpha'%joint], f[1]['%s_beta'%joint], f[1]['%s_gamma'%joint]] for f in r.iterrows()] # Make sure the columsn are organized in xyz order
+                rot_order = track.skeleton[joint]['order']
+                euler_rots = np.array(R.from_rotvec(expmap).as_euler(rot_order.lower(), degrees=True))
+                
+                # Create the corresponding columns in the new DataFrame    
+                euler_df['%s_%srotation'%(joint, rot_order[0])] = pd.Series(data=[e[0] for e in euler_rots], index=euler_df.index)
+                euler_df['%s_%srotation'%(joint, rot_order[1])] = pd.Series(data=[e[1] for e in euler_rots], index=euler_df.index)
+                euler_df['%s_%srotation'%(joint, rot_order[2])] = pd.Series(data=[e[2] for e in euler_rots], index=euler_df.index)
+
+            new_track = track.clone()
+            new_track.values = euler_df
+            Q.append(new_track)
+
+        return Q
+
+class Mirror(BaseEstimator, TransformerMixin):
+    def __init__(self, axis="X", append=True):
+        """
+        Mirrors the data 
+        """
+        self.axis = axis
+        self.append = append
+        
+    
+    def fit(self, X, y=None):
+        return self
+    
+    def transform(self, X, y=None):
+        print("Mirror: " + self.axis)
+        Q = []
+        
+        if self.append:
+            for track in X:
+                Q.append(track)
+            
+        for track in X:
+            channels = []
+            titles = []
+            
+            if self.axis == "X":
+                signs = np.array([1,-1,-1])
+            if self.axis == "Y":
+                signs = np.array([-1,1,-1])
+            if self.axis == "Z":
+                signs = np.array([-1,-1,1])
+
+            euler_df = track.values
+
+            # Create a new DataFrame to store the exponential map rep
+            new_df = pd.DataFrame(index=euler_df.index)
+
+            # Copy the root positions into the new DataFrame
+            rxp = '%s_Xposition'%track.root_name
+            ryp = '%s_Yposition'%track.root_name
+            rzp = '%s_Zposition'%track.root_name
+            new_df[rxp] = pd.Series(data=-signs[0]*euler_df[rxp], index=new_df.index)
+            new_df[ryp] = pd.Series(data=-signs[1]*euler_df[ryp], index=new_df.index)
+            new_df[rzp] = pd.Series(data=-signs[2]*euler_df[rzp], index=new_df.index)
+            
+            # List the columns that contain rotation channels
+            rots = [c for c in euler_df.columns if ('rotation' in c and 'Nub' not in c)]
+            lft_joints = (joint for joint in track.skeleton if '_l_' in joint and 'Nub' not in joint)
+            rgt_joints = (joint for joint in track.skeleton if '_r_' in joint and 'Nub' not in joint)
+                        
+            new_track = track.clone()
+
+            for lft_joint in lft_joints:                
+                rgt_joint = lft_joint.replace('_l_', '_r_')
+                
+                # Create the corresponding columns in the new DataFrame                
+                new_df['%s_Xrotation'%lft_joint] = pd.Series(data=signs[0]*track.values['%s_Xrotation'%rgt_joint], index=new_df.index)
+                new_df['%s_Yrotation'%lft_joint] = pd.Series(data=signs[1]*track.values['%s_Yrotation'%rgt_joint], index=new_df.index)
+                new_df['%s_Zrotation'%lft_joint] = pd.Series(data=signs[2]*track.values['%s_Zrotation'%rgt_joint], index=new_df.index)
+                
+                new_df['%s_Xrotation'%rgt_joint] = pd.Series(data=signs[0]*track.values['%s_Xrotation'%lft_joint], index=new_df.index)
+                new_df['%s_Yrotation'%rgt_joint] = pd.Series(data=signs[1]*track.values['%s_Yrotation'%lft_joint], index=new_df.index)
+                new_df['%s_Zrotation'%rgt_joint] = pd.Series(data=signs[2]*track.values['%s_Zrotation'%lft_joint], index=new_df.index)
+    
+            # List the joints that are not left or right, i.e. are on the trunk
+            joints = (joint for joint in track.skeleton if 'Nub' not in joint and '_l_' not in joint and '_r_' not in joint)
+
+            for joint in joints:
+                # Create the corresponding columns in the new DataFrame
+                new_df['%s_Xrotation'%joint] = pd.Series(data=signs[0]*track.values['%s_Xrotation'%joint], index=new_df.index)
+                new_df['%s_Yrotation'%joint] = pd.Series(data=signs[1]*track.values['%s_Yrotation'%joint], index=new_df.index)
+                new_df['%s_Zrotation'%joint] = pd.Series(data=signs[2]*track.values['%s_Zrotation'%joint], index=new_df.index)
+
+            new_track.values = new_df
+            Q.append(new_track)
+
+        return Q
+
+    def inverse_transform(self, X, copy=None, start_pos=None):
+        return X
+
+class JointSelector(BaseEstimator, TransformerMixin):
+    '''
+    Allows for filtering the mocap data to include only the selected joints
+    '''
+    def __init__(self, joints, include_root=False):
+        self.joints = joints
+        self.include_root = include_root
+
+    def fit(self, X, y=None):
+        selected_joints = []
+        selected_channels = []
+
+        if self.include_root:
+            selected_joints.append(X[0].root_name)
+        
+        selected_joints.extend(self.joints)
+
+        for joint_name in selected_joints:
+            selected_channels.extend([o for o in X[0].values.columns if ((joint_name + "_X") in o or (joint_name + "_Y") in o or (joint_name + "_Z") in o) and 'Nub' not in o and 'Null' not in o])
+        
+        self.selected_joints = selected_joints
+        self.selected_channels = selected_channels
+        self.not_selected = X[0].values.columns.difference(selected_channels)
+        self.not_selected_values = {c:X[0].values[c].values[0] for c in self.not_selected}
+
+        self.orig_skeleton = X[0].skeleton
+        return self
+
+    def transform(self, X, y=None):
+        print("JointSelector")
+        Q = []
+
+        for track in X:
+            t2 = track.clone()
+            
+            for key in track.skeleton.keys():
+                if key not in self.selected_joints:
+                    t2.skeleton.pop(key)
+            t2.values = track.values[self.selected_channels]
+
+            Q.append(t2)
+      
+
+        return Q
+    
+    def inverse_transform(self, X, copy=None):
+        Q = []
+
+        for track in X:
+            t2 = track.clone()
+            t2.skeleton = self.orig_skeleton
+            for d in self.not_selected:
+                t2.values[d] = self.not_selected_values[d]
+            Q.append(t2)
+
+        return Q
+
+
+class Numpyfier(BaseEstimator, TransformerMixin):
+    '''
+    Just converts the values in a MocapData object into a numpy array
+    Useful for the final stage of a pipeline before training
+    '''
+    def __init__(self):
+        pass
+
+    def fit(self, X, y=None):
+        self.org_mocap_ = X[0].clone()
+        self.org_mocap_.values.drop(self.org_mocap_.values.index, inplace=True)
+
+        return self
+
+    def transform(self, X, y=None):
+        print("Numpyfier")
+        Q = []
+        
+        for track in X:
+            Q.append(track.values.values)
+            #print("Numpyfier:" + str(track.values.columns))
+            
+        return np.array(Q)
+
+    def inverse_transform(self, X, copy=None):
+        Q = []
+
+        for track in X:
+            
+            new_mocap = self.org_mocap_.clone()
+            time_index = pd.to_timedelta([f for f in range(track.shape[0])], unit='s')
+
+            new_df =  pd.DataFrame(data=track, index=time_index, columns=self.org_mocap_.values.columns)
+            
+            new_mocap.values = new_df
+            
+
+            Q.append(new_mocap)
+
+        return Q
+    
+class Slicer(BaseEstimator, TransformerMixin):
+    '''
+    Slice the data into intervals of equal size 
+    '''
+    def __init__(self, window_size, overlap=0.5):
+        self.window_size = window_size
+        self.overlap = overlap
+        pass
+
+    def fit(self, X, y=None):
+        self.org_mocap_ = X[0].clone()
+        self.org_mocap_.values.drop(self.org_mocap_.values.index, inplace=True)
+
+        return self
+
+    def transform(self, X, y=None):
+        print("Slicer")
+        Q = []
+        
+        for track in X:
+            vals = track.values.values
+            nframes = vals.shape[0]
+            overlap_frames = (int)(self.overlap*self.window_size)
+            
+            n_sequences = (nframes-overlap_frames)//(self.window_size-overlap_frames)
+            
+            if n_sequences>0:
+                y = np.zeros((n_sequences, self.window_size, vals.shape[1]))
+
+                # extract sequences from the input data
+                for i in range(0,n_sequences):
+                    frameIdx = (self.window_size-overlap_frames) * i
+                    Q.append(vals[frameIdx:frameIdx+self.window_size,:])
+
+        return np.array(Q)
+
+    def inverse_transform(self, X, copy=None):
+        Q = []
+
+        for track in X:
+            
+            new_mocap = self.org_mocap_.clone()
+            time_index = pd.to_timedelta([f for f in range(track.shape[0])], unit='s')
+
+            new_df =  pd.DataFrame(data=track, index=time_index, columns=self.org_mocap_.values.columns)
+            
+            new_mocap.values = new_df
+            
+
+            Q.append(new_mocap)
+
+        return Q
+
+class RootTransformer(BaseEstimator, TransformerMixin):
+    def __init__(self, method, position_smoothing=0, rotation_smoothing=0):
+        """
+        Accepted methods:
+            abdolute_translation_deltas
+            pos_rot_deltas
+        """
+        self.method = method
+        self.position_smoothing=position_smoothing
+        self.rotation_smoothing=rotation_smoothing
+    
+    def fit(self, X, y=None):
+        return self
+    
+    def transform(self, X, y=None):
+        print("RootTransformer")
+        Q = []
+
+        for track in X:
+            if self.method == 'abdolute_translation_deltas':
+                new_df = track.values.copy()
+                xpcol = '%s_Xposition'%track.root_name
+                ypcol = '%s_Yposition'%track.root_name
+                zpcol = '%s_Zposition'%track.root_name
+
+
+                dxpcol = '%s_dXposition'%track.root_name
+                dzpcol = '%s_dZposition'%track.root_name
+                
+                x=track.values[xpcol].copy()
+                z=track.values[zpcol].copy()
+                
+                if self.position_smoothing>0:
+                    x_sm = filters.gaussian_filter1d(x, self.position_smoothing, axis=0, mode='nearest')    
+                    z_sm = filters.gaussian_filter1d(z, self.position_smoothing, axis=0, mode='nearest')                    
+                    dx = pd.Series(data=x_sm, index=new_df.index).diff()
+                    dz = pd.Series(data=z_sm, index=new_df.index).diff()
+                    new_df[xpcol] = x-x_sm
+                    new_df[zpcol] = z-z_sm
+                else:
+                    dx = x.diff()
+                    dz = z.diff()
+                    new_df.drop([xpcol, zpcol], axis=1, inplace=True)
+                    
+                dx[0] = dx[1]
+                dz[0] = dz[1]
+                
+                new_df[dxpcol] = dx
+                new_df[dzpcol] = dz
+                
+                new_track = track.clone()
+                new_track.values = new_df
+            # end of abdolute_translation_deltas
+            
+            elif self.method == 'hip_centric':
+                new_track = track.clone()
+
+                # Absolute columns
+                xp_col = '%s_Xposition'%track.root_name
+                yp_col = '%s_Yposition'%track.root_name
+                zp_col = '%s_Zposition'%track.root_name
+
+                xr_col = '%s_Xrotation'%track.root_name
+                yr_col = '%s_Yrotation'%track.root_name
+                zr_col = '%s_Zrotation'%track.root_name
+                
+                new_df = track.values.copy()
+
+                all_zeros = np.zeros(track.values[xp_col].values.shape)
+
+                new_df[xp_col] = pd.Series(data=all_zeros, index=new_df.index)
+                new_df[yp_col] = pd.Series(data=all_zeros, index=new_df.index)
+                new_df[zp_col] = pd.Series(data=all_zeros, index=new_df.index)
+
+                new_df[xr_col] = pd.Series(data=all_zeros, index=new_df.index)
+                new_df[yr_col] = pd.Series(data=all_zeros, index=new_df.index)
+                new_df[zr_col] = pd.Series(data=all_zeros, index=new_df.index)
+
+                new_track.values = new_df
+
+            #print(new_track.values.columns)
+            Q.append(new_track)
+
+        return Q
+
+    def inverse_transform(self, X, copy=None, start_pos=None):
+        Q = []
+
+        #TODO: simplify this implementation
+
+        startx = 0
+        startz = 0
+
+        if start_pos is not None:
+            startx, startz = start_pos
+
+        for track in X:
+            new_track = track.clone()
+            if self.method == 'abdolute_translation_deltas':
+                new_df = new_track.values
+                xpcol = '%s_Xposition'%track.root_name
+                ypcol = '%s_Yposition'%track.root_name
+                zpcol = '%s_Zposition'%track.root_name
+
+
+                dxpcol = '%s_dXposition'%track.root_name
+                dzpcol = '%s_dZposition'%track.root_name
+
+                dx = track.values[dxpcol].values
+                dz = track.values[dzpcol].values
+
+                recx = [startx]
+                recz = [startz]
+
+                for i in range(dx.shape[0]-1):
+                    recx.append(recx[i]+dx[i+1])
+                    recz.append(recz[i]+dz[i+1])
+
+                # recx = [recx[i]+dx[i+1] for i in range(dx.shape[0]-1)]
+                # recz = [recz[i]+dz[i+1] for i in range(dz.shape[0]-1)]
+                # recx = dx[:-1] + dx[1:]
+                # recz = dz[:-1] + dz[1:]
+                if self.position_smoothing > 0:                    
+                    new_df[xpcol] = pd.Series(data=new_df[xpcol]+recx, index=new_df.index)
+                    new_df[zpcol] = pd.Series(data=new_df[zpcol]+recz, index=new_df.index)
+                else:
+                    new_df[xpcol] = pd.Series(data=recx, index=new_df.index)
+                    new_df[zpcol] = pd.Series(data=recz, index=new_df.index)
+
+                new_df.drop([dxpcol, dzpcol], axis=1, inplace=True)
+                
+                new_track.values = new_df
+            # end of abdolute_translation_deltas
+            
+            Q.append(new_track)
+
+        return Q
+
+
+class RootNormalizer(BaseEstimator, TransformerMixin):
+    """
+    Make subjects in TalkingWithHands16.2M face the same direction
+    This class is not for general uses. Only compatible to GENEA 2022 challenge dataset
+    Added by Youngwoo Yoon, April 2022
+    """
+    def __init__(self):
+        pass
+
+    def fit(self, X, y=None):
+        return self
+
+    def transform(self, X, y=None):
+        print("RootNormalizer")
+        Q = []
+
+        for track in X:
+            new_track = track.clone()
+
+            xp_col = '%s_Xposition'%track.root_name
+            yp_col = '%s_Yposition'%track.root_name
+            zp_col = '%s_Zposition'%track.root_name
+
+            xr_col = '%s_Xrotation'%track.root_name
+            yr_col = '%s_Yrotation'%track.root_name
+            zr_col = '%s_Zrotation'%track.root_name
+
+            new_df = track.values.copy()
+
+            all_zeros = np.zeros(track.values[xp_col].values.shape)
+            mean_xp = np.mean(track.values[xp_col].values)
+            mean_yp = np.mean(track.values[yp_col].values)
+            mean_zp = np.mean(track.values[zp_col].values)
+
+            if track.values[xp_col].values[0] < 0:
+                new_yr = np.full(track.values[xp_col].values.shape, -90)
+            else:
+                new_yr = np.full(track.values[xp_col].values.shape, 90)
+
+            new_df[xp_col] = pd.Series(data=track.values[xp_col]-mean_xp, index=new_df.index)
+            new_df[yp_col] = pd.Series(data=track.values[yp_col]-mean_yp, index=new_df.index)
+            new_df[zp_col] = pd.Series(data=track.values[zp_col]-mean_zp, index=new_df.index)
+
+            new_df[xr_col] = pd.Series(data=all_zeros, index=new_df.index)
+            new_df[yr_col] = pd.Series(data=new_yr, index=new_df.index)
+            new_df[zr_col] = pd.Series(data=all_zeros, index=new_df.index)
+
+            new_track.values = new_df
+
+            Q.append(new_track)
+
+        return Q
+
+    def inverse_transform(self, X, copy=None):
+        # NOT IMPLEMENTED
+        return X
+
+
+class RootCentricPositionNormalizer(BaseEstimator, TransformerMixin):
+    def __init__(self):
+        pass
+
+    def fit(self, X, y=None):
+        return self
+
+    def transform(self, X, y=None):
+        Q = []
+
+        for track in X:
+            new_track = track.clone()
+
+            rxp = '%s_Xposition'%track.root_name
+            ryp = '%s_Yposition'%track.root_name
+            rzp = '%s_Zposition'%track.root_name
+
+            projected_root_pos = track.values[[rxp, ryp, rzp]]
+
+            projected_root_pos.loc[:,ryp] = 0 # we want the root's projection on the floor plane as the ref
+
+            new_df = pd.DataFrame(index=track.values.index)
+
+            all_but_root = [joint for joint in track.skeleton if track.root_name not in joint]
+            # all_but_root = [joint for joint in track.skeleton]
+            for joint in all_but_root:                
+                new_df['%s_Xposition'%joint] = pd.Series(data=track.values['%s_Xposition'%joint]-projected_root_pos[rxp], index=new_df.index)
+                new_df['%s_Yposition'%joint] = pd.Series(data=track.values['%s_Yposition'%joint]-projected_root_pos[ryp], index=new_df.index)
+                new_df['%s_Zposition'%joint] = pd.Series(data=track.values['%s_Zposition'%joint]-projected_root_pos[rzp], index=new_df.index)
+            
+
+            # keep the root as it is now
+            new_df[rxp] = track.values[rxp]
+            new_df[ryp] = track.values[ryp]
+            new_df[rzp] = track.values[rzp]
+
+            new_track.values = new_df
+
+            Q.append(new_track)
+        
+        return Q
+
+    def inverse_transform(self, X, copy=None):
+        Q = []
+
+        for track in X:
+            new_track = track.clone()
+
+            rxp = '%s_Xposition'%track.root_name
+            ryp = '%s_Yposition'%track.root_name
+            rzp = '%s_Zposition'%track.root_name
+
+            projected_root_pos = track.values[[rxp, ryp, rzp]]
+
+            projected_root_pos.loc[:,ryp] = 0 # we want the root's projection on the floor plane as the ref
+
+            new_df = pd.DataFrame(index=track.values.index)
+
+            for joint in track.skeleton:                
+                new_df['%s_Xposition'%joint] = pd.Series(data=track.values['%s_Xposition'%joint]+projected_root_pos[rxp], index=new_df.index)
+                new_df['%s_Yposition'%joint] = pd.Series(data=track.values['%s_Yposition'%joint]+projected_root_pos[ryp], index=new_df.index)
+                new_df['%s_Zposition'%joint] = pd.Series(data=track.values['%s_Zposition'%joint]+projected_root_pos[rzp], index=new_df.index)
+                
+
+            new_track.values = new_df
+
+            Q.append(new_track)
+        
+        return Q
+
+class Flattener(BaseEstimator, TransformerMixin):
+    def __init__(self):
+        pass
+
+    def fit(self, X, y=None):
+        return self
+
+    def transform(self, X, y=None):
+        return np.concatenate(X, axis=0)
+
+class ConstantsRemover(BaseEstimator, TransformerMixin):
+    '''
+    For now it just looks at the first track
+    '''
+
+    def __init__(self, eps = 1e-6):
+        self.eps = eps
+        
+
+    def fit(self, X, y=None):
+        stds = X[0].values.std()
+        cols = X[0].values.columns.values
+        self.const_dims_ = [c for c in cols if (stds[c] < self.eps).any()]
+        self.const_values_ = {c:X[0].values[c].values[0] for c in cols if (stds[c] < self.eps).any()}
+        return self
+
+    def transform(self, X, y=None):
+        Q = []
+        
+
+        for track in X:
+            t2 = track.clone()
+            #for key in t2.skeleton.keys():
+            #    if key in self.ConstDims_:
+            #        t2.skeleton.pop(key)
+            #print(track.values.columns.difference(self.const_dims_))
+            t2.values.drop(self.const_dims_, axis=1, inplace=True)
+            #t2.values = track.values[track.values.columns.difference(self.const_dims_)]
+            Q.append(t2)
+        
+        return Q
+    
+    def inverse_transform(self, X, copy=None):
+        Q = []
+        
+        for track in X:
+            t2 = track.clone()
+            for d in self.const_dims_:
+                t2.values[d] = self.const_values_[d]
+#                t2.values.assign(d=pd.Series(data=self.const_values_[d], index = t2.values.index))
+            Q.append(t2)
+
+        return Q
+
+class ListStandardScaler(BaseEstimator, TransformerMixin):
+    def __init__(self, is_DataFrame=False):
+        self.is_DataFrame = is_DataFrame
+    
+    def fit(self, X, y=None):
+        if self.is_DataFrame:
+            X_train_flat = np.concatenate([m.values for m in X], axis=0)
+        else:
+            X_train_flat = np.concatenate([m for m in X], axis=0)
+
+        self.data_mean_ = np.mean(X_train_flat, axis=0)
+        self.data_std_ = np.std(X_train_flat, axis=0)
+
+        return self
+    
+    def transform(self, X, y=None):
+        Q = []
+        
+        for track in X:
+            if self.is_DataFrame:
+                normalized_track = track.copy()
+                normalized_track.values = (track.values - self.data_mean_) / self.data_std_
+            else:
+                normalized_track = (track - self.data_mean_) / self.data_std_
+
+            Q.append(normalized_track)
+        
+        if self.is_DataFrame:
+            return Q
+        else:
+            return np.array(Q)
+
+    def inverse_transform(self, X, copy=None):
+        Q = []
+        
+        for track in X:
+            
+            if self.is_DataFrame:
+                unnormalized_track = track.copy()
+                unnormalized_track.values = (track.values * self.data_std_) + self.data_mean_
+            else:
+                unnormalized_track = (track * self.data_std_) + self.data_mean_
+
+            Q.append(unnormalized_track)
+        
+        if self.is_DataFrame:
+            return Q
+        else:
+            return np.array(Q)
+
+class ListMinMaxScaler(BaseEstimator, TransformerMixin):
+    def __init__(self, is_DataFrame=False):
+        self.is_DataFrame = is_DataFrame
+    
+    def fit(self, X, y=None):
+        if self.is_DataFrame:
+            X_train_flat = np.concatenate([m.values for m in X], axis=0)
+        else:
+            X_train_flat = np.concatenate([m for m in X], axis=0)
+
+        self.data_max_ = np.max(X_train_flat, axis=0)
+        self.data_min_ = np.min(X_train_flat, axis=0)
+
+        return self
+    
+    def transform(self, X, y=None):
+        Q = []
+        
+        for track in X:
+            if self.is_DataFrame:
+                normalized_track = track.copy()
+                normalized_track.values = (track.values - self.data_min_) / (self.data_max_ - self.data_min_) 
+            else:
+                normalized_track = (track - self.data_min_) / (self.data_max_ - self.data_min_)
+
+            Q.append(normalized_track)
+        
+        if self.is_DataFrame:
+            return Q
+        else:
+            return np.array(Q)
+
+    def inverse_transform(self, X, copy=None):
+        Q = []
+        
+        for track in X:
+            
+            if self.is_DataFrame:
+                unnormalized_track = track.copy()
+                unnormalized_track.values = (track.values * (self.data_max_ - self.data_min_)) + self.data_min_
+            else:
+                unnormalized_track = (track * (self.data_max_ - self.data_min_)) + self.data_min_
+
+            Q.append(unnormalized_track)
+        
+        if self.is_DataFrame:
+            return Q
+        else:
+            return np.array(Q)
+        
+class DownSampler(BaseEstimator, TransformerMixin):
+    def __init__(self, tgt_fps, keep_all=True):
+        self.tgt_fps = tgt_fps
+        self.keep_all = keep_all
+        
+    
+    def fit(self, X, y=None):    
+
+        return self
+    
+    def transform(self, X, y=None):
+        Q = []
+        
+        for track in X:
+            orig_fps=round(1.0/track.framerate)
+            rate = orig_fps//self.tgt_fps
+            if orig_fps%self.tgt_fps!=0:
+                print("error orig_fps (" + str(orig_fps) + ") is not dividable with tgt_fps (" + str(self.tgt_fps) + ")")
+            else:
+                print("downsampling with rate: " + str(rate))
+                
+            #print(track.values.size)
+            for ii in range(0,rate):
+                new_track = track.clone()
+                new_track.values = track.values[ii:-1:rate].copy()            
+                #print(new_track.values.size)
+                #new_track = track[0:-1:self.rate]
+                new_track.framerate = 1.0/self.tgt_fps
+                Q.append(new_track)
+                if not self.keep_all:
+                    break
+        
+        return Q
+        
+    def inverse_transform(self, X, copy=None):
+      return X
+
+class ReverseTime(BaseEstimator, TransformerMixin):
+    def __init__(self, append=True):
+        self.append = append
+        
+    
+    def fit(self, X, y=None):    
+
+        return self
+    
+    def transform(self, X, y=None):
+        Q = []
+        if self.append:
+            for track in X:
+                Q.append(track)
+        for track in X:
+            new_track = track.clone()                            
+            new_track.values = track.values[-1::-1]
+            Q.append(new_track)
+        
+        return Q
+        
+    def inverse_transform(self, X, copy=None):
+      return X
+
+#TODO: JointsSelector (x)
+#TODO: SegmentMaker
+#TODO: DynamicFeaturesAdder
+#TODO: ShapeFeaturesAdder
+#TODO: DataFrameNumpier (x)
+
+class TemplateTransform(BaseEstimator, TransformerMixin):
+    def __init__(self):
+        pass
+
+    def fit(self, X, y=None):
+        return self
+
+    def transform(self, X, y=None):
+        return X
+
diff --git a/scripts/pymo/rotation_tools.py b/scripts/pymo/rotation_tools.py
new file mode 100644
index 0000000..ee4a4f4
--- /dev/null
+++ b/scripts/pymo/rotation_tools.py
@@ -0,0 +1,220 @@
+'''
+Tools for Manipulating and Converting 3D Rotations
+
+By Omid Alemi
+Created: June 12, 2017
+
+Adapted from that matlab file...
+'''
+
+import math
+import numpy as np
+import transforms3d as t3d
+from scipy.spatial.transform import Rotation as R
+
+def deg2rad(x):
+    return x/180*math.pi
+
+
+def rad2deg(x):
+    return x/math.pi*180
+
+def unroll(rots):
+
+    new_rots = rots.copy()
+    
+    # Compute angles and alternative rotation angles
+    angs = np.linalg.norm(rots, axis=1)
+    dotprod = np.einsum('ij,ij->i', rots[:-1,:], rots[1:,:])
+    alt_angs=2*np.pi-angs
+
+    #find discontinuities
+    d_angs = np.diff(angs, axis=0)
+    d_angs2 = alt_angs[1:]-angs[:-1]
+    
+    # check if dot product is <0
+    swps = np.where((dotprod<-1))[0]
+    #print(np.sum(swps))
+    #swps = np.where((np.abs(d_ax)>0.5))[0]
+    #swps = np.where(np.abs(d_angs2)<np.abs(d_angs))[0]
+
+    #reshape into intervals where we should unroll the rotations
+    isodd = swps.shape[0] % 2 == 1
+    if isodd:
+        swps=swps[:-1]
+        #swps = np.append(swps, rots.shape[0]-1)
+    intv = 1+swps.reshape((swps.shape[0]//2, 2))
+    for ii in range(intv.shape[0]):
+        new_ax = -rots[intv[ii,0]:intv[ii,1],:]/np.tile(angs[intv[ii,0]:intv[ii,1], None], (1,3))
+        new_angs = alt_angs[intv[ii,0]:intv[ii,1]]
+        new_rots[intv[ii,0]:intv[ii,1],:] = new_ax*np.tile(new_angs[:, None], (1,3))
+
+    return new_rots
+    
+def euler2expmap(rot, order='XYZ',use_deg=False):
+    if use_deg:
+        rot = np.deg2rad(rot)
+    #print("rot:" + str(rot))
+    vec, theta = t3d.euler.euler2axangle(rot[0], rot[1], rot[2], 'r' + order.lower())
+    return vec*theta
+
+def expmap2euler(rot, order='XYZ',use_deg=False):
+    theta = np.linalg.norm(rot)
+    if theta > 1.0e-10:
+        vector = rot / theta
+    else:
+        vector = np.array([1.,0.,0.])
+        theta=0.0
+    eul = t3d.euler.axangle2euler(vector, theta, 'r' + order.lower())
+    if use_deg:
+        return np.rad2deg(eul)
+    else:
+        return eul
+
+class Rotation():
+    def __init__(self,rot, param_type, **params):
+        self.rotmat = []
+        if param_type == 'euler':
+            self._from_euler(rot[0],rot[1],rot[2], params)
+        elif param_type == 'expmap':
+            self._from_expmap(rot[0], rot[1], rot[2], params)
+
+    def _from_euler(self, alpha, beta, gamma, params):
+        '''Expecting degress'''
+        
+        if params['from_deg']==True:
+            alpha = deg2rad(alpha)
+            beta = deg2rad(beta)
+            gamma = deg2rad(gamma)
+            
+        order = "s" + ((params['order']).lower())[::-1]
+#        Quaternions.from_euler()        
+        self.rotmat = np.transpose(t3d.euler.euler2mat(gamma, beta , alpha, axes=order))
+
+#        ca = math.cos(alpha)
+#        cb = math.cos(beta)
+#        cg = math.cos(gamma)
+#        sa = math.sin(alpha)
+#        sb = math.sin(beta)
+#        sg = math.sin(gamma)        
+#
+#        Rx = np.asarray([[1, 0, 0], 
+#              [0, ca, sa], 
+#              [0, -sa, ca]
+#              ])
+#
+#        Ry = np.asarray([[cb, 0, -sb], 
+#              [0, 1, 0],
+#              [sb, 0, cb]])
+#
+#        Rz = np.asarray([[cg, sg, 0],
+#              [-sg, cg, 0],
+#              [0, 0, 1]])
+#
+#        self.rotmat = np.eye(3)
+#        
+#        order = params['order']
+#        for i in range(0,len(order)):
+#            if order[i]=='X':
+#                self.rotmat = np.matmul(Rx, self.rotmat)
+#            elif order[i]=='Y':
+#                self.rotmat = np.matmul(Ry, self.rotmat)
+#            elif order[i]=='Z':
+#                self.rotmat = np.matmul(Rz, self.rotmat)
+#            else:
+#                print('unknown rotation axis: ' + order[i])
+#                
+#        # self.rotmat = np.matmul(np.matmul(Rz, Ry), Rx)
+#        print ("------" + "TRUE")
+#        print (self.rotmat)
+   
+    def _from_expmap(self, alpha, beta, gamma, params):
+        if (alpha == 0 and beta == 0 and gamma == 0):
+            self.rotmat = np.eye(3)
+            return
+
+        #TODO: Check exp map params
+
+        theta = np.linalg.norm([alpha, beta, gamma])
+
+        expmap = [alpha, beta, gamma] / theta
+
+        x = expmap[0]
+        y = expmap[1]
+        z = expmap[2]
+
+        s = math.sin(theta/2)
+        c = math.cos(theta/2)
+
+        self.rotmat = np.asarray([
+            [2*(x**2-1)*s**2+1,  2*x*y*s**2-2*z*c*s,  2*x*z*s**2+2*y*c*s],
+            [2*x*y*s**2+2*z*c*s,  2*(y**2-1)*s**2+1,  2*y*z*s**2-2*x*c*s],
+            [2*x*z*s**2-2*y*c*s, 2*y*z*s**2+2*x*c*s , 2*(z**2-1)*s**2+1]
+        ])
+        
+
+
+    def get_euler_axis(self):
+        R = self.rotmat
+        theta = math.acos((self.rotmat.trace() - 1) / 2)
+        axis = np.asarray([R[2,1] - R[1,2], R[0,2] - R[2,0], R[1,0] - R[0,1]])
+        axis = axis/(2*math.sin(theta))
+        return theta, axis
+
+    def to_expmap(self):
+        axis, theta = t3d.axangles.mat2axangle(self.rotmat, unit_thresh=1e-05)
+#        theta, axis = self.get_euler_axis()
+        rot_arr = theta * axis
+        if np.isnan(rot_arr).any():
+            rot_arr = [0, 0, 0]
+        return rot_arr
+    
+    def to_euler(self, use_deg=False, order='xyz'):
+        order = "s" + order.lower()
+        eulers = t3d.euler.mat2euler(np.transpose(self.rotmat), axes=order)
+        return eulers[::-1]
+        
+#        eulers = np.zeros((2, 3))
+#
+#        if np.absolute(np.absolute(self.rotmat[2, 0]) - 1) < 1e-12:
+#            #GIMBAL LOCK!
+#            print('Gimbal')
+#            if np.absolute(self.rotmat[2, 0]) - 1 < 1e-12:
+#                eulers[:,0] = math.atan2(-self.rotmat[0,1], -self.rotmat[0,2])
+#                eulers[:,1] = -math.pi/2
+#            else:
+#                eulers[:,0] = math.atan2(self.rotmat[0,1], -elf.rotmat[0,2])
+#                eulers[:,1] = math.pi/2
+#            
+#            return eulers
+#
+#        theta = - math.asin(self.rotmat[2,0])
+#        theta2 = math.pi - theta
+#
+#        # psi1, psi2
+#        eulers[0,0] = math.atan2(self.rotmat[2,1]/math.cos(theta), self.rotmat[2,2]/math.cos(theta))
+#        eulers[1,0] = math.atan2(self.rotmat[2,1]/math.cos(theta2), self.rotmat[2,2]/math.cos(theta2))
+#
+#        # theta1, theta2
+#        eulers[0,1] = theta
+#        eulers[1,1] = theta2
+#
+#        # phi1, phi2
+#        eulers[0,2] = math.atan2(self.rotmat[1,0]/math.cos(theta), self.rotmat[0,0]/math.cos(theta))
+#        eulers[1,2] = math.atan2(self.rotmat[1,0]/math.cos(theta2), self.rotmat[0,0]/math.cos(theta2))
+#
+        if use_deg:
+            eulers = rad2deg(eulers)
+
+        return eulers
+    
+    def to_quat(self):
+        #TODO
+        pass
+    
+    def __str__(self):
+        return "Rotation Matrix: \n " + self.rotmat.__str__()
+    
+
+
+
diff --git a/scripts/pymo/viz_tools.py b/scripts/pymo/viz_tools.py
new file mode 100644
index 0000000..be325be
--- /dev/null
+++ b/scripts/pymo/viz_tools.py
@@ -0,0 +1,235 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import os
+
+def save_fig(fig_id, tight_layout=True):
+    if tight_layout:
+        plt.tight_layout()
+    plt.savefig(fig_id + '.png', format='png', dpi=300)
+    
+    
+def draw_stickfigure(mocap_track, frame, data=None, joints=None, draw_names=False, ax=None, figsize=(8,8)):
+    if ax is None:
+        fig = plt.figure(figsize=figsize)
+        ax = fig.add_subplot(111)
+    
+    if joints is None:
+        joints_to_draw = mocap_track.skeleton.keys()
+    else:
+        joints_to_draw = joints
+    
+    if data is None:
+        df = mocap_track.values
+    else:
+        df = data
+        
+    for joint in joints_to_draw:
+        ax.scatter(x=df['%s_Xposition'%joint][frame], 
+                   y=df['%s_Yposition'%joint][frame],  
+                   alpha=0.6, c='b', marker='o')
+
+        parent_x = df['%s_Xposition'%joint][frame]
+        parent_y = df['%s_Yposition'%joint][frame]
+        
+        children_to_draw = [c for c in mocap_track.skeleton[joint]['children'] if c in joints_to_draw]
+        
+        for c in children_to_draw:
+            child_x = df['%s_Xposition'%c][frame]
+            child_y = df['%s_Yposition'%c][frame]
+            ax.plot([parent_x, child_x], [parent_y, child_y], 'k-', lw=2)
+            
+        if draw_names:
+            ax.annotate(joint, 
+                    (df['%s_Xposition'%joint][frame] + 0.1, 
+                     df['%s_Yposition'%joint][frame] + 0.1))
+
+    return ax
+
+def draw_stickfigure3d(mocap_track, frame, data=None, joints=None, draw_names=False, ax=None, figsize=(8,8)):
+    from mpl_toolkits.mplot3d import Axes3D
+    
+    if ax is None:
+        fig = plt.figure(figsize=figsize)
+        ax = fig.add_subplot(111, projection='3d') 
+    
+    if joints is None:
+        joints_to_draw = mocap_track.skeleton.keys()
+    else:
+        joints_to_draw = joints
+    
+    if data is None:
+        df = mocap_track.values
+    else:
+        df = data
+        
+    for joint in joints_to_draw:
+        parent_x = df['%s_Xposition'%joint][frame]
+        parent_y = df['%s_Zposition'%joint][frame]
+        parent_z = df['%s_Yposition'%joint][frame]
+        # ^ In mocaps, Y is the up-right axis 
+
+        ax.scatter(xs=parent_x, 
+                   ys=parent_y,  
+                   zs=parent_z,  
+                   alpha=0.6, c='b', marker='o')
+
+        
+        children_to_draw = [c for c in mocap_track.skeleton[joint]['children'] if c in joints_to_draw]
+        
+        for c in children_to_draw:
+            child_x = df['%s_Xposition'%c][frame]
+            child_y = df['%s_Zposition'%c][frame]
+            child_z = df['%s_Yposition'%c][frame]
+            # ^ In mocaps, Y is the up-right axis
+
+            ax.plot([parent_x, child_x], [parent_y, child_y], [parent_z, child_z], 'k-', lw=2, c='black')
+            
+        if draw_names:
+            ax.text(x=parent_x + 0.1, 
+                    y=parent_y + 0.1,
+                    z=parent_z + 0.1,
+                    s=joint,
+                    color='rgba(0,0,0,0.9')
+
+    return ax
+
+
+def sketch_move(mocap_track, data=None, ax=None, figsize=(16,8)):
+    if ax is None:
+        fig = plt.figure(figsize=figsize)
+        ax = fig.add_subplot(111)
+    
+    if data is None:
+        data = mocap_track.values
+
+    for frame in range(0, data.shape[0], 4):
+#         draw_stickfigure(mocap_track, f, data=data, ax=ax)
+        
+        for joint in mocap_track.skeleton.keys():
+            children_to_draw = [c for c in mocap_track.skeleton[joint]['children']]
+            
+            parent_x = data['%s_Xposition'%joint][frame]
+            parent_y = data['%s_Yposition'%joint][frame]
+            
+            frame_alpha = frame/data.shape[0]
+            
+            for c in children_to_draw:
+                child_x = data['%s_Xposition'%c][frame]
+                child_y = data['%s_Yposition'%c][frame]
+                
+                ax.plot([parent_x, child_x], [parent_y, child_y], '-', lw=1, color='gray', alpha=frame_alpha)
+
+
+
+def viz_cnn_filter(feature_to_viz, mocap_track, data, gap=25):
+    fig = plt.figure(figsize=(16,4))
+    ax = plt.subplot2grid((1,8),(0,0))
+    ax.imshow(feature_to_viz.T, aspect='auto', interpolation='nearest')
+    
+    ax = plt.subplot2grid((1,8),(0,1), colspan=7)
+    for frame in range(feature_to_viz.shape[0]):
+        frame_alpha = 0.2#frame/data.shape[0] * 2 + 0.2
+
+        for joint_i, joint in enumerate(mocap_track.skeleton.keys()):
+            children_to_draw = [c for c in mocap_track.skeleton[joint]['children']]
+
+            parent_x = data['%s_Xposition'%joint][frame] + frame * gap
+            parent_y = data['%s_Yposition'%joint][frame] 
+
+            ax.scatter(x=parent_x, 
+                       y=parent_y,  
+                       alpha=0.6,
+                       cmap='RdBu',
+                       c=feature_to_viz[frame][joint_i] * 10000,
+                       marker='o',
+                       s = abs(feature_to_viz[frame][joint_i] * 10000))
+            plt.axis('off')
+            for c in children_to_draw:
+                child_x = data['%s_Xposition'%c][frame] + frame * gap
+                child_y = data['%s_Yposition'%c][frame] 
+
+                ax.plot([parent_x, child_x], [parent_y, child_y], '-', lw=1, color='gray', alpha=frame_alpha)
+
+                   
+def print_skel(X):
+    stack = [X.root_name]
+    tab=0
+    while stack:
+        joint = stack.pop()
+        tab = len(stack)
+        print('%s- %s (%s)'%('| '*tab, joint, X.skeleton[joint]['parent']))
+        for c in X.skeleton[joint]['children']:
+            stack.append(c)
+
+
+def nb_play_mocap_fromurl(mocap, mf, frame_time=1/30, scale=1, base_url='http://titan:8385'):
+    if mf == 'bvh':
+        bw = BVHWriter()
+        with open('test.bvh', 'w') as ofile:
+            bw.write(mocap, ofile)
+        
+        filepath = '../notebooks/test.bvh'
+    elif mf == 'pos':
+        c = list(mocap.values.columns)
+
+        for cc in c:
+            if 'rotation' in cc:
+                c.remove(cc)
+        mocap.values.to_csv('test.csv', index=False, columns=c)
+        
+        filepath = '../notebooks/test.csv'
+    else:
+        return
+    
+    url = '%s/mocapplayer/player.html?data_url=%s&scale=%f&cz=200&order=xzyi&frame_time=%f'%(base_url, filepath, scale, frame_time)
+    iframe = '<iframe src=' + url + ' width="100%" height=500></iframe>'
+    link = '<a href=%s target="_blank">New Window</a>'%url
+    return IPython.display.HTML(iframe+link)
+
+def nb_play_mocap(mocap, mf, meta=None, frame_time=1/30, scale=1, camera_z=500, base_url=None):
+    data_template = 'var dataBuffer = `$$DATA$$`;'
+    data_template += 'var metadata = $$META$$;'
+    data_template += 'start(dataBuffer, metadata, $$CZ$$, $$SCALE$$, $$FRAMETIME$$);'
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+
+
+    if base_url is None:
+        base_url = os.path.join(dir_path, 'mocapplayer/playBuffer.html')
+    
+    # print(dir_path)
+
+    if mf == 'bvh':
+        pass
+    elif mf == 'pos':
+        cols = list(mocap.values.columns)
+        for c in cols:
+            if 'rotation' in c:
+                cols.remove(c)
+        
+        data_csv = mocap.values.to_csv(index=False, columns=cols)
+
+        if meta is not None:
+            lines = [','.join(item) for item in meta.astype('str')]
+            meta_csv = '[' + ','.join('[%s]'%l for l in lines) +']'            
+        else:
+            meta_csv = '[]'
+        
+        data_assigned = data_template.replace('$$DATA$$', data_csv)
+        data_assigned = data_assigned.replace('$$META$$', meta_csv)
+        data_assigned = data_assigned.replace('$$CZ$$', str(camera_z))
+        data_assigned = data_assigned.replace('$$SCALE$$', str(scale))
+        data_assigned = data_assigned.replace('$$FRAMETIME$$', str(frame_time))
+
+    else:
+        return
+    
+    
+
+    with open(os.path.join(dir_path, 'mocapplayer/data.js'), 'w') as oFile:
+        oFile.write(data_assigned)
+
+    url = '%s?&cz=200&order=xzyi&frame_time=%f&scale=%f'%(base_url, frame_time, scale)
+    iframe = '<iframe frameborder="0" src=' + url + ' width="100%" height=500></iframe>'
+    link = '<a href=%s target="_blank">New Window</a>'%url
+    return IPython.display.HTML(iframe+link)
\ No newline at end of file
diff --git a/scripts/pymo/writers.py b/scripts/pymo/writers.py
new file mode 100644
index 0000000..630dbdf
--- /dev/null
+++ b/scripts/pymo/writers.py
@@ -0,0 +1,70 @@
+import numpy as np
+import pandas as pd
+
+class BVHWriter():
+    def __init__(self):
+        pass
+    
+    def write(self, X, ofile, framerate=-1):
+        
+        # Writing the skeleton info
+        ofile.write('HIERARCHY\n')
+        
+        self.motions_ = []
+        self._printJoint(X, X.root_name, 0, ofile)
+
+        # Writing the motion header
+        ofile.write('MOTION\n')
+        ofile.write('Frames: %d\n'%X.values.shape[0])
+        
+        if framerate > 0:
+            ofile.write('Frame Time: %f\n'%float(1.0/framerate))
+        else:
+            ofile.write('Frame Time: %f\n'%X.framerate)
+
+        # Writing the data
+        self.motions_ = np.asarray(self.motions_).T
+        lines = [" ".join(item) for item in self.motions_.astype(str)]
+        ofile.write("".join("%s\n"%l for l in lines))
+
+    def _printJoint(self, X, joint, tab, ofile):
+        
+        if X.skeleton[joint]['parent'] == None:
+            ofile.write('ROOT %s\n'%joint)
+        elif len(X.skeleton[joint]['children']) > 0:
+            ofile.write('%sJOINT %s\n'%('\t'*(tab), joint))
+        else:
+            ofile.write('%sEnd site\n'%('\t'*(tab)))
+
+        ofile.write('%s{\n'%('\t'*(tab)))
+        
+        ofile.write('%sOFFSET %3.5f %3.5f %3.5f\n'%('\t'*(tab+1),
+                                                X.skeleton[joint]['offsets'][0],
+                                                X.skeleton[joint]['offsets'][1],
+                                                X.skeleton[joint]['offsets'][2]))
+        rot_order = X.skeleton[joint]['order']
+        
+        #print("rot_order = " + rot_order)
+        channels = X.skeleton[joint]['channels']
+        rot = [c for c in channels if ('rotation' in c)]
+        pos = [c for c in channels if ('position' in c)]
+        
+        n_channels = len(rot) +len(pos)
+        ch_str = ''
+        if n_channels > 0:
+            for ci in range(len(pos)):
+                cn = pos[ci]
+                self.motions_.append(np.asarray(X.values['%s_%s'%(joint,cn)].values))
+                ch_str = ch_str + ' ' + cn 
+            for ci in range(len(rot)):
+                cn = '%srotation'%(rot_order[ci])
+                self.motions_.append(np.asarray(X.values['%s_%s'%(joint,cn)].values))
+                ch_str = ch_str + ' ' + cn 
+        if len(X.skeleton[joint]['children']) > 0:
+            #ch_str = ''.join(' %s'*n_channels%tuple(channels))
+            ofile.write('%sCHANNELS %d%s\n' %('\t'*(tab+1), n_channels, ch_str)) 
+
+            for c in X.skeleton[joint]['children']:
+                self._printJoint(X, c, tab+1, ofile)
+
+        ofile.write('%s}\n'%('\t'*(tab)))
diff --git a/scripts/save_clustered_fast.py b/scripts/save_clustered_fast.py
new file mode 100644
index 0000000..42cf378
--- /dev/null
+++ b/scripts/save_clustered_fast.py
@@ -0,0 +1,201 @@
+import argparse
+import glob
+import math
+import os
+import pickle
+import pprint
+from pathlib import Path
+
+import librosa
+import numpy as np
+import time
+
+import torch
+from scipy.signal import savgol_filter
+import joblib as jl
+
+import utils
+from pymo.preprocessing import *
+from pymo.viz_tools import *
+from pymo.writers import *
+from tqdm import tqdm
+from utils.data_utils import SubtitleWrapper, normalize_string
+from utils.train_utils import set_logger
+
+from data_loader.data_preprocessor import DataPreprocessor
+
+from trinity_data_to_lmdb import process_bvh
+
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+
+def generate_gestures(args, DAE, rnn, bvh_file):
+
+    poses, poses_mirror = process_bvh(bvh_file)
+
+
+    mean = np.array(args.data_mean).squeeze()
+    std = np.array(args.data_std).squeeze()
+    std = np.clip(std, a_min=0.01, a_max=None)
+    out_poses = (poses_mirror - mean) / std
+
+    target = torch.from_numpy(out_poses)
+    # target = torch.unsqueeze(target,2)
+    target = target.to(device).float()
+    reconstructed = []
+    # for i in range(len(out_poses)):
+    #     input = torch.unsqueeze(target[i],0)
+    #     current_out = pose_decoder(input)
+    #     reconstructed.append(current_out)
+    encoded = DAE.encoder(target)
+    # encoded = torch.squeeze(encoded, 2)
+    # encoded = encoded.to('cpu')
+    # encoded = encoded.detach().numpy()
+    all_frames_from_rnn = None
+    for i in range(0, len(encoded), args.n_poses):
+        input_seq = encoded[i:i+args.n_poses]
+        input_pre_seq = encoded[i]
+        output_seq = encoded[i:i+args.n_poses]
+
+        input_seq = torch.unsqueeze(input_seq, 0)
+        input_seq = input_seq.transpose(0, 1)
+
+        output_seq = torch.unsqueeze(output_seq, 0)
+        output_seq = output_seq.transpose(0, 1)
+        reconstructed_rnn = torch.zeros(args.n_poses, output_seq.size(1), rnn.decoder.output_size).to(output_seq.device)
+        # run words through encoder
+        encoder_outputs, encoder_hidden = rnn.encoder(input_seq, None)
+        decoder_hidden = encoder_hidden[:rnn.decoder.n_layers]  # use last hidden state from encoder
+
+        # run through decoder one time step at a time
+        decoder_input = output_seq[0]  # initial pose from the dataset
+        reconstructed_rnn[0] = decoder_input
+
+        for t in range(1, rnn.n_frames):
+            decoder_output, decoder_hidden, _ = rnn.decoder(None, decoder_input, decoder_hidden, encoder_outputs,
+                                                             None)
+            reconstructed_rnn[t] = decoder_output
+
+            if t < rnn.n_pre_poses:
+                decoder_input = output_seq[t]  # next input is current target
+            else:
+                decoder_input = decoder_output  # next input is current prediction
+        if all_frames_from_rnn == None:
+            all_frames_from_rnn = reconstructed_rnn.transpose(0,1)
+        else:
+            all_frames_from_rnn = torch.cat((all_frames_from_rnn, reconstructed_rnn.transpose(0, 1)), 1)
+
+
+    #     Todo: decode DAE
+    all_frames_from_rnn = torch.squeeze(all_frames_from_rnn, 0)
+    reconstructed_seq_DAE = DAE.decoder(all_frames_from_rnn)
+    # reconstructed_seq_DAE = torch.squeeze(reconstructed_seq_DAE, 2)
+    reconstructed_seq_DAE = reconstructed_seq_DAE.to('cpu')
+    reconstructed_seq_DAE = reconstructed_seq_DAE.detach().numpy()
+
+
+    return out_poses, np.array(reconstructed_seq_DAE)
+
+    '''
+        # smoothing motion transition
+        if len(out_list) > 0:
+            last_poses = out_list[-1][-args.n_pre_poses:]
+            out_list[-1] = out_list[-1][:-args.n_pre_poses]  # delete the last part
+
+            for j in range(len(last_poses)):
+                n = len(last_poses)
+                prev = last_poses[j]
+                next = out_seq[j]
+                out_seq[j] = prev * (n - j) / (n + 1) + next * (j + 1) / (n + 1)
+
+        out_list.append(out_seq)
+
+    print('Avg. inference time: {:.2} s'.format((time.time() - start) / num_subdivision))
+
+    # aggregate results
+    out_poses = np.vstack(out_list)
+    '''
+    return out_poses
+
+
+
+def main(checkpoint_path_DAE, checkpoint_path_rnn):
+    args, DAE, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+        checkpoint_path_DAE, device)
+    args, rnn, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+        checkpoint_path_rnn, device)
+    pprint.pprint(vars(args))
+    save_path = '../output/infer_sample'
+    os.makedirs(save_path, exist_ok=True)
+
+    # load lang_model
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    with open(vocab_cache_path, 'rb') as f:
+        lang_model = pickle.load(f)
+
+    # prepare input
+    # transcript = SubtitleWrapper(transcript_path).get()
+
+    k = 0
+    kmeans_labels = pickle.load(open('../output/kmeans_labels.bin', 'rb'))
+    # inference
+
+    gesture_path = '/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Training_data/Motion'
+
+    bvh_files = sorted(glob.glob(gesture_path + "/*.bvh"))
+    for bvh_file in tqdm(bvh_files):
+        name = os.path.split(bvh_file)[1][:-4]
+        print("Processing", name)
+        org_poses, reconstructed = generate_gestures(args, DAE, rnn, bvh_file)
+
+        # unnormalize
+        mean = np.array(args.data_mean).squeeze()
+        std = np.array(args.data_std).squeeze()
+        std = np.clip(std, a_min=0.01, a_max=None)
+        reconstructed = np.multiply(reconstructed, std) + mean
+        org_poses = np.multiply(org_poses, std) + mean
+
+        for j in range(0, len(reconstructed), args.n_poses):
+            save_path = '../output/clusters/' + str(kmeans_labels[k])
+            k = k+1
+            # make a BVH
+            filename_prefix = '{}'.format("test_original")
+            make_bvh(save_path, filename_prefix, org_poses[j:j+args.n_poses])
+            filename_prefix = '{}'.format("test_reconstructed")
+            make_bvh(save_path, filename_prefix, reconstructed[j:j+args.n_poses])
+
+
+def make_bvh(save_path, filename_prefix, poses):
+    writer = BVHWriter()
+    pipeline = jl.load('../resource/data_pipe.sav')
+
+    # smoothing
+    n_poses = poses.shape[0]
+    out_poses = np.zeros((n_poses, poses.shape[1]))
+
+    for i in range(poses.shape[1]):
+        out_poses[:, i] = savgol_filter(poses[:, i], 15, 2)  # NOTE: smoothing on rotation matrices is not optimal
+
+    # rotation matrix to euler angles
+    out_poses = out_poses.reshape((out_poses.shape[0], -1, 9))
+    out_poses = out_poses.reshape((out_poses.shape[0], out_poses.shape[1], 3, 3))
+    out_euler = np.zeros((out_poses.shape[0], out_poses.shape[1] * 3))
+    for i in range(out_poses.shape[0]):  # frames
+        r = R.from_matrix(out_poses[i])
+        out_euler[i] = r.as_euler('ZXY', degrees=True).flatten()
+
+    bvh_data = pipeline.inverse_transform([out_euler])
+
+    out_bvh_path = os.path.join(save_path, filename_prefix + '_generated.bvh')
+    with open(out_bvh_path, 'w') as f:
+        writer.write(bvh_data[0], f)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("ckpt_path_DAE", type=Path)
+    parser.add_argument("ckpt_path_Autoencode", type=Path)
+    args = parser.parse_args()
+    labels = pickle.load(open('../output/kmeans_labels.bin', 'rb'))
+    main(args.ckpt_path_DAE, args.ckpt_path_Autoencode)
diff --git a/scripts/test_pymo.py b/scripts/test_pymo.py
new file mode 100644
index 0000000..00c2261
--- /dev/null
+++ b/scripts/test_pymo.py
@@ -0,0 +1,138 @@
+import argparse
+import glob
+import os
+from pathlib import Path
+
+import librosa
+import lmdb
+import pyarrow
+from sklearn.pipeline import Pipeline
+import joblib as jl
+
+from pymo.preprocessing import *
+from pymo.parsers import BVHParser
+from utils.data_utils import SubtitleWrapper, normalize_string
+
+# 18 joints (only upper body)
+target_joints = ['b_spine0', 'b_spine1', 'b_spine2', 'b_spine3', 'b_l_shoulder', 'b_l_arm', 'b_l_arm_twist', 'b_l_forearm', 'b_l_wrist_twist', 'b_l_wrist', 'b_r_shoulder', 'b_r_arm', 'b_r_arm_twist', 'b_r_forearm', 'b_r_wrist_twist', 'b_r_wrist', 'b_neck0', 'b_head']
+
+# 24 joints (upper and lower body excluding fingers)
+# target_joints = ['body_world', 'b_root', 'b_l_upleg', 'b_l_leg', 'b_r_upleg', 'b_r_leg', 'b_spine0', 'b_spine1', 'b_spine2', 'b_spine3', 'b_l_shoulder', 'b_l_arm', 'b_l_arm_twist', 'b_l_forearm', 'b_l_wrist_twist', 'b_l_wrist', 'b_r_shoulder', 'b_r_arm', 'b_r_arm_twist', 'b_r_forearm', 'b_r_wrist_twist', 'b_r_wrist', 'b_neck0', 'b_head']
+
+# 56 joints (upper and lower body including fingers)
+# target_joints = ['body_world', 'b_root', 'b_l_upleg', 'b_l_leg', 'b_r_upleg', 'b_r_leg', 'b_spine0', 'b_spine1', 'b_spine2', 'b_spine3', 'b_l_shoulder', 'b_l_arm', 'b_l_arm_twist', 'b_l_forearm', 'b_l_wrist_twist', 'b_l_wrist', 'b_l_pinky1', 'b_l_pinky2', 'b_l_pinky3', 'b_l_ring1', 'b_l_ring2', 'b_l_ring3', 'b_l_middle1', 'b_l_middle2', 'b_l_middle3', 'b_l_index1', 'b_l_index2', 'b_l_index3', 'b_l_thumb0', 'b_l_thumb1', 'b_l_thumb2', 'b_l_thumb3', 'b_r_shoulder', 'b_r_arm', 'b_r_arm_twist', 'b_r_forearm', 'b_r_wrist_twist', 'b_r_wrist', 'b_r_thumb0', 'b_r_thumb1', 'b_r_thumb2', 'b_r_thumb3', 'b_r_pinky1', 'b_r_pinky2', 'b_r_pinky3', 'b_r_middle1', 'b_r_middle2', 'b_r_middle3', 'b_r_ring1', 'b_r_ring2', 'b_r_ring3', 'b_r_index1', 'b_r_index2', 'b_r_index3', 'b_neck0', 'b_head']
+
+
+target_joints_trinity = ['Spine', 'Spine1', 'Spine2', 'Spine3', 'Neck', 'Neck1', 'Head',
+                 'RightShoulder', 'RightArm', 'RightForeArm', 'RightHand',
+                 'LeftShoulder', 'LeftArm', 'LeftForeArm', 'LeftHand']
+
+bvh_file_triity = '/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Motion/TestSeq001.bvh'
+bvh_file_twh = '/local-scratch/pjomeyaz/rosie_gesture_benchmark/cloned/Clustering/must/GENEA/Co-Speech_Gesture_Generation/dataset/dataset_v1/val/bvh/val_2022_v1_030.bvh'
+
+
+def process_bvh_trinity(gesture_filename):
+    p = BVHParser()
+
+    data_all = list()
+    data_all.append(p.parse(gesture_filename))
+
+    data_pipe = Pipeline([
+        ('dwnsampl', DownSampler(tgt_fps=20, keep_all=False)),
+        ('root', RootTransformer('hip_centric')),
+        ('mir', Mirror(axis='X', append=True)),
+        ('jtsel', JointSelector(target_joints_trinity, include_root=True)),
+        ('cnst', ConstantsRemover()),
+        ('np', Numpyfier())
+    ])
+
+    data_pipe = Pipeline([
+        ('dwnsampl', DownSampler(tgt_fps=20, keep_all=False)),
+        ('root', RootNormalizer()),
+        ('jtsel', JointSelector(target_joints, include_root=False)),
+        # ('mir', Mirror(axis='X', append=True)),
+        ('cnst', ConstantsRemover()),
+        ('np', Numpyfier())
+    ])
+
+    out_data = data_pipe.fit_transform(data_all)
+    jl.dump(data_pipe, os.path.join('../resource', 'data_pipe.sav'))
+
+    # euler -> rotation matrix
+    out_data = out_data.reshape((out_data.shape[0], out_data.shape[1], -1, 3))
+    out_matrix = np.zeros((out_data.shape[0], out_data.shape[1], out_data.shape[2], 9))
+    for i in range(out_data.shape[0]):  # mirror
+        for j in range(out_data.shape[1]):  # frames
+            r = R.from_euler('ZXY', out_data[i, j], degrees=True)
+            out_matrix[i, j] = r.as_matrix().reshape(out_data.shape[2], 9)
+    out_matrix = out_matrix.reshape((out_data.shape[0], out_data.shape[1], -1))
+
+    return out_matrix[0], out_matrix[1]
+
+def process_bvh_twh(gesture_filename, dump_pipeline=False):
+    p = BVHParser()
+
+    data_all = list()
+    data_all.append(p.parse(gesture_filename))
+
+    data_pipe = Pipeline([
+        ('dwnsampl', DownSampler(tgt_fps=20, keep_all=False)),
+        ('root', RootNormalizer()),
+        ('jtsel', JointSelector(target_joints, include_root=False)),
+        # ('mir', Mirror(axis='X', append=True)),
+        ('np', Numpyfier())
+    ])
+
+    out_data = data_pipe.fit_transform(data_all)
+    if dump_pipeline:
+        jl.dump(data_pipe, os.path.join('../resource', 'data_pipe.sav'))
+
+    # euler -> rotation matrix
+    out_data = out_data.reshape((out_data.shape[0], out_data.shape[1], -1, 6))  # 3 pos (XYZ), 3 rot (ZXY)
+    out_matrix = np.zeros((out_data.shape[0], out_data.shape[1], out_data.shape[2], 12))  # 3 pos, 1 rot matrix (9 elements)
+    for i in range(out_data.shape[0]):  # mirror
+        for j in range(out_data.shape[1]):  # frames
+            for k in range(out_data.shape[2]):  # joints
+                out_matrix[i, j, k, :3] = out_data[i, j, k, :3]  # positions
+                r = R.from_euler('ZXY', out_data[i, j, k, 3:], degrees=True)
+                out_matrix[i, j, k, 3:] = r.as_matrix().flatten()  # rotations
+    out_matrix = out_matrix.reshape((out_data.shape[0], out_data.shape[1], -1))
+
+    return out_matrix[0]
+
+def process_bvh_rot_only(gesture_filename, dump_pipeline=False):
+    p = BVHParser()
+
+    data_all = list()
+    data_all.append(p.parse(gesture_filename))
+
+    data_pipe = Pipeline([
+        ('dwnsampl', DownSampler(tgt_fps=20, keep_all=False)),
+        # ('root', RootTransformer('hip_centric')),
+        ('root', RootNormalizer()),
+        # ('mir', Mirror(axis='X', append=True)),
+        ('jtsel', JointSelector(target_joints, include_root=True)),
+        # ('cnst', ConstantsRemover()),
+        # ('np', Numpyfier())
+    ])
+
+    out_data = data_pipe.fit_transform(data_all)
+    jl.dump(data_pipe, os.path.join('../resource', 'data_pipe.sav'))
+
+    # euler -> rotation matrix
+    out_data = out_data.reshape((out_data.shape[0], out_data.shape[1], -1, 3))
+    out_matrix = np.zeros((out_data.shape[0], out_data.shape[1], out_data.shape[2], 9))
+    for i in range(out_data.shape[0]):  # mirror
+        for j in range(out_data.shape[1]):  # frames
+            r = R.from_euler('ZXY', out_data[i, j], degrees=True)
+            out_matrix[i, j] = r.as_matrix().reshape(out_data.shape[2], 9)
+    out_matrix = out_matrix.reshape((out_data.shape[0], out_data.shape[1], -1))
+
+    return out_matrix[0], out_matrix[1]
+
+
+# trinity_a = process_bvh_trinity(bvh_file_triity)
+trinity_a = process_bvh_trinity(bvh_file_twh)
+
+twh_a = process_bvh_twh(bvh_file_twh)
+print("Test")
\ No newline at end of file
diff --git a/scripts/test_transformers/transformers_model.py b/scripts/test_transformers/transformers_model.py
new file mode 100644
index 0000000..bab4067
--- /dev/null
+++ b/scripts/test_transformers/transformers_model.py
@@ -0,0 +1,1092 @@
+# -*- coding: utf-8 -*-
+"""6 - Attention is All You Need.ipynb
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/github/bentrevett/pytorch-seq2seq/blob/master/6%20-%20Attention%20is%20All%20You%20Need.ipynb
+
+# 6 - Attention is All You Need
+
+In this notebook we will be implementing a (slightly modified version) of the Transformer model from the [Attention is All You Need](https://arxiv.org/abs/1706.03762) paper. All images in this notebook will be taken from the Transformer paper. For more information about the Transformer, [see](https://www.mihaileric.com/posts/transformers-attention-in-disguise/) [these](https://jalammar.github.io/illustrated-transformer/) [three](http://nlp.seas.harvard.edu/2018/04/03/attention.html) articles.
+
+![](https://github.com/bentrevett/pytorch-seq2seq/blob/master/assets/transformer1.png?raw=1)
+
+## Introduction
+
+Similar to the Convolutional Sequence-to-Sequence model, the Transformer does not use any recurrence. It also does not use any convolutional layers. Instead the model is entirely made up of linear layers, attention mechanisms and normalization.
+
+As of January 2020, Transformers are the dominant architecture in NLP and are used to achieve state-of-the-art results for many tasks and it appears as if they will be for the near future.
+
+The most popular Transformer variant is [BERT](https://arxiv.org/abs/1810.04805) (**B**idirectional **E**ncoder **R**epresentations from **T**ransformers) and pre-trained versions of BERT are commonly used to replace the embedding layers - if not more - in NLP models.
+
+A common library used when dealing with pre-trained transformers is the [Transformers](https://huggingface.co/transformers/) library, see [here](https://huggingface.co/transformers/pretrained_models.html) for a list of all pre-trained models available.
+
+The differences between the implementation in this notebook and the paper are:
+- we use a learned positional encoding instead of a static one
+- we use the standard Adam optimizer with a static learning rate instead of one with warm-up and cool-down steps
+- we do not use label smoothing
+
+We make all of these changes as they closely follow BERT's set-up and the majority of Transformer variants use a similar set-up.
+
+## Preparing the Data
+
+As always, let's import all the required modules and set the random seeds for reproducability.
+"""
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+
+import torchtext
+from torchtext.legacy.datasets import Multi30k
+from torchtext.legacy.data import Field, BucketIterator
+
+import matplotlib.pyplot as plt
+import matplotlib.ticker as ticker
+
+import spacy
+import numpy as np
+
+import random
+import math
+import time
+
+SEED = 1234
+
+random.seed(SEED)
+np.random.seed(SEED)
+torch.manual_seed(SEED)
+torch.cuda.manual_seed(SEED)
+torch.backends.cudnn.deterministic = True
+
+"""We'll then create our tokenizers as before."""
+
+spacy_de = spacy.load('de_core_news_sm')
+spacy_en = spacy.load('en_core_web_sm')
+
+
+def tokenize_de(text):
+    """
+    Tokenizes German text from a string into a list of strings
+    """
+    return [tok.text for tok in spacy_de.tokenizer(text)]
+
+
+def tokenize_en(text):
+    """
+    Tokenizes English text from a string into a list of strings
+    """
+    return [tok.text for tok in spacy_en.tokenizer(text)]
+
+
+"""Our fields are the same as the previous notebook. The model expects data to be fed in with the batch dimension first, so we use `batch_first = True`. """
+
+SRC = Field(tokenize=tokenize_de,
+            init_token='<sos>',
+            eos_token='<eos>',
+            lower=True,
+            batch_first=True)
+
+TRG = Field(tokenize=tokenize_en,
+            init_token='<sos>',
+            eos_token='<eos>',
+            lower=True,
+            batch_first=True)
+
+"""We then load the Multi30k dataset and build the vocabulary."""
+
+train_data, valid_data, test_data = Multi30k.splits(exts=('.de', '.en'),
+                                                    fields=(SRC, TRG))
+
+SRC.build_vocab(train_data, min_freq=2)
+TRG.build_vocab(train_data, min_freq=2)
+
+"""Finally, we define the device and the data iterator."""
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+BATCH_SIZE = 128
+
+train_iterator, valid_iterator, test_iterator = BucketIterator.splits(
+    (train_data, valid_data, test_data),
+    batch_size=BATCH_SIZE,
+    device=device)
+
+"""## Building the Model
+
+Next, we'll build the model. Like previous notebooks it is made up of an *encoder* and a *decoder*, with the encoder *encoding* the input/source sentence (in German) into *context vector* and the decoder then *decoding* this context vector to output our output/target sentence (in English). 
+
+### Encoder
+
+Similar to the ConvSeq2Seq model, the Transformer's encoder does not attempt to compress the entire source sentence, $X = (x_1, ... ,x_n)$, into a single context vector, $z$. Instead it produces a sequence of context vectors, $Z = (z_1, ... , z_n)$. So, if our input sequence was 5 tokens long we would have $Z = (z_1, z_2, z_3, z_4, z_5)$. Why do we call this a sequence of context vectors and not a sequence of hidden states? A hidden state at time $t$ in an RNN has only seen tokens $x_t$ and all the tokens before it. However, each context vector here has seen all tokens at all positions within the input sequence.
+
+![](https://github.com/bentrevett/pytorch-seq2seq/blob/master/assets/transformer-encoder.png?raw=1)
+
+First, the tokens are passed through a standard embedding layer. Next, as the model has no recurrent it has no idea about the order of the tokens within the sequence. We solve this by using a second embedding layer called a *positional embedding layer*. This is a standard embedding layer where the input is not the token itself but the position of the token within the sequence, starting with the first token, the `<sos>` (start of sequence) token, in position 0. The position embedding has a "vocabulary" size of 100, which means our model can accept sentences up to 100 tokens long. This can be increased if we want to handle longer sentences.
+
+The original Transformer implementation from the Attention is All You Need paper does not learn positional embeddings. Instead it uses a fixed static embedding. Modern Transformer architectures, like BERT, use positional embeddings instead, hence we have decided to use them in these tutorials. Check out [this](http://nlp.seas.harvard.edu/2018/04/03/attention.html#positional-encoding) section to read more about the positional embeddings used in the original Transformer model.
+
+Next, the token and positional embeddings are elementwise summed together to get a vector which contains information about the token and also its position with in the sequence. However, before they are summed, the token embeddings are multiplied by a scaling factor which is $\sqrt{d_{model}}$, where $d_{model}$ is the hidden dimension size, `hid_dim`. This supposedly reduces variance in the embeddings and the model is difficult to train reliably without this scaling factor. Dropout is then applied to the combined embeddings.
+
+The combined embeddings are then passed through $N$ *encoder layers* to get $Z$, which is then output and can be used by the decoder.
+
+The source mask, `src_mask`, is simply the same shape as the source sentence but has a value of 1 when the token in the source sentence is not a `<pad>` token and 0 when it is a `<pad>` token. This is used in the encoder layers to mask the multi-head attention mechanisms, which are used to calculate and apply attention over the source sentence, so the model does not pay attention to `<pad>` tokens, which contain no useful information.
+"""
+
+
+class Encoder(nn.Module):
+    def __init__(self,
+                 input_dim,
+                 hid_dim,
+                 n_layers,
+                 n_heads,
+                 pf_dim,
+                 dropout,
+                 device,
+                 max_length=100):
+        super().__init__()
+
+        self.device = device
+
+        self.tok_embedding = nn.Embedding(input_dim, hid_dim)
+        self.pos_embedding = nn.Embedding(max_length, hid_dim)
+
+        self.layers = nn.ModuleList([EncoderLayer(hid_dim,
+                                                  n_heads,
+                                                  pf_dim,
+                                                  dropout,
+                                                  device)
+                                     for _ in range(n_layers)])
+
+        self.dropout = nn.Dropout(dropout)
+
+        self.scale = torch.sqrt(torch.FloatTensor([hid_dim])).to(device)
+
+    def forward(self, src, src_mask):
+        # src = [batch size, src len]
+        # src_mask = [batch size, 1, 1, src len]
+
+        batch_size = src.shape[0]
+        src_len = src.shape[1]
+
+        pos = torch.arange(0, src_len).unsqueeze(0).repeat(batch_size, 1).to(self.device)
+
+        # pos = [batch size, src len]
+
+        src = self.dropout((self.tok_embedding(src) * self.scale) + self.pos_embedding(pos))
+
+        # src = [batch size, src len, hid dim]
+
+        for layer in self.layers:
+            src = layer(src, src_mask)
+
+        # src = [batch size, src len, hid dim]
+
+        return src
+
+
+"""### Encoder Layer
+
+The encoder layers are where all of the "meat" of the encoder is contained. We first pass the source sentence and its mask into the *multi-head attention layer*, then perform dropout on it, apply a residual connection and pass it through a [Layer Normalization](https://arxiv.org/abs/1607.06450) layer. We then pass it through a *position-wise feedforward* layer and then, again, apply dropout, a residual connection and then layer normalization to get the output of this layer which is fed into the next layer. The parameters are not shared between layers. 
+
+The mutli head attention layer is used by the encoder layer to attend to the source sentence, i.e. it is calculating and applying attention over itself instead of another sequence, hence we call it *self attention*.
+
+[This](https://mlexplained.com/2018/01/13/weight-normalization-and-layer-normalization-explained-normalization-in-deep-learning-part-2/) article goes into more detail about layer normalization, but the gist is that it normalizes the values of the features, i.e. across the hidden dimension, so each feature has a mean of 0 and a standard deviation of 1. This allows neural networks with a larger number of layers, like the Transformer, to be trained easier.
+"""
+
+
+class EncoderLayer(nn.Module):
+    def __init__(self,
+                 hid_dim,
+                 n_heads,
+                 pf_dim,
+                 dropout,
+                 device):
+        super().__init__()
+
+        self.self_attn_layer_norm = nn.LayerNorm(hid_dim)
+        self.ff_layer_norm = nn.LayerNorm(hid_dim)
+        self.self_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)
+        self.positionwise_feedforward = PositionwiseFeedforwardLayer(hid_dim,
+                                                                     pf_dim,
+                                                                     dropout)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, src, src_mask):
+        # src = [batch size, src len, hid dim]
+        # src_mask = [batch size, 1, 1, src len]
+
+        # self attention
+        _src, _ = self.self_attention(src, src, src, src_mask)
+
+        # dropout, residual connection and layer norm
+        src = self.self_attn_layer_norm(src + self.dropout(_src))
+
+        # src = [batch size, src len, hid dim]
+
+        # positionwise feedforward
+        _src = self.positionwise_feedforward(src)
+
+        # dropout, residual and layer norm
+        src = self.ff_layer_norm(src + self.dropout(_src))
+
+        # src = [batch size, src len, hid dim]
+
+        return src
+
+
+"""### Mutli Head Attention Layer
+
+One of the key, novel concepts introduced by the Transformer paper is the *multi-head attention layer*. 
+
+![](https://github.com/bentrevett/pytorch-seq2seq/blob/master/assets/transformer-attention.png?raw=1)
+
+Attention can be though of as *queries*, *keys* and *values* - where the query is used with the key to get an attention vector (usually the output of a *softmax* operation and has all values between 0 and 1 which sum to 1) which is then used to get a weighted sum of the values.
+
+The Transformer uses *scaled dot-product attention*, where the query and key are combined by taking the dot product between them, then applying the softmax operation and scaling by $d_k$ before finally then multiplying by the value. $d_k$ is the *head dimension*, `head_dim`, which we will shortly explain further.
+
+$$ \text{Attention}(Q, K, V) = \text{Softmax} \big( \frac{QK^T}{\sqrt{d_k}} \big)V $$ 
+
+This is similar to standard *dot product attention* but is scaled by $d_k$, which the paper states is used to stop the results of the dot products growing large, causing gradients to become too small.
+
+However, the scaled dot-product attention isn't simply applied to the queries, keys and values. Instead of doing a single attention application the queries, keys and values have their `hid_dim` split into $h$ *heads* and the scaled dot-product attention is calculated over all heads in parallel. This means instead of paying attention to one concept per attention application, we pay attention to $h$. We then re-combine the heads into their `hid_dim` shape, thus each `hid_dim` is potentially paying attention to $h$ different concepts.
+
+$$ \text{MultiHead}(Q, K, V) = \text{Concat}(\text{head}_1,...,\text{head}_h)W^O $$
+
+$$\text{head}_i = \text{Attention}(QW_i^Q, KW_i^K, VW_i^V) $$
+
+$W^O$ is the linear layer applied at the end of the multi-head attention layer, `fc`. $W^Q, W^K, W^V$ are the linear layers `fc_q`, `fc_k` and `fc_v`.
+
+Walking through the module, first we calculate $QW^Q$, $KW^K$ and $VW^V$ with the linear layers, `fc_q`, `fc_k` and `fc_v`, to give us `Q`, `K` and `V`. Next, we split the `hid_dim` of the query, key and value into `n_heads` using `.view` and correctly permute them so they can be multiplied together. We then calculate the `energy` (the un-normalized attention) by multiplying `Q` and `K` together and scaling it by the square root of `head_dim`, which is calulated as `hid_dim // n_heads`. We then mask the energy so we do not pay attention over any elements of the sequeuence we shouldn't, then apply the softmax and dropout. We then apply the attention to the value heads, `V`, before combining the `n_heads` together. Finally, we multiply this $W^O$, represented by `fc_o`. 
+
+Note that in our implementation the lengths of the keys and values are always the same, thus when matrix multiplying the output of the softmax, `attention`, with `V` we will always have valid dimension sizes for matrix multiplication. This multiplication is carried out using `torch.matmul` which, when both tensors are >2-dimensional, does a batched matrix multiplication over the last two dimensions of each tensor. This will be a **[query len, key len] x [value len, head dim]** batched matrix multiplication over the batch size and each head which provides the **[batch size, n heads, query len, head dim]** result.
+
+One thing that looks strange at first is that dropout is applied directly to the attention. This means that our attention vector will most probably not sum to 1 and we may pay full attention to a token but the attention over that token is set to 0 by dropout. This is never explained, or even mentioned, in the paper however is used by the [official implementation](https://github.com/tensorflow/tensor2tensor/) and every Transformer implementation since, [including BERT](https://github.com/google-research/bert/).
+"""
+
+
+class MultiHeadAttentionLayer(nn.Module):
+    def __init__(self, hid_dim, n_heads, dropout, device):
+        super().__init__()
+
+        assert hid_dim % n_heads == 0
+
+        self.hid_dim = hid_dim
+        self.n_heads = n_heads
+        self.head_dim = hid_dim // n_heads
+
+        self.fc_q = nn.Linear(hid_dim, hid_dim)
+        self.fc_k = nn.Linear(hid_dim, hid_dim)
+        self.fc_v = nn.Linear(hid_dim, hid_dim)
+
+        self.fc_o = nn.Linear(hid_dim, hid_dim)
+
+        self.dropout = nn.Dropout(dropout)
+
+        self.scale = torch.sqrt(torch.FloatTensor([self.head_dim])).to(device)
+
+    def forward(self, query, key, value, mask=None):
+        batch_size = query.shape[0]
+
+        # query = [batch size, query len, hid dim]
+        # key = [batch size, key len, hid dim]
+        # value = [batch size, value len, hid dim]
+
+        Q = self.fc_q(query)
+        K = self.fc_k(key)
+        V = self.fc_v(value)
+
+        # Q = [batch size, query len, hid dim]
+        # K = [batch size, key len, hid dim]
+        # V = [batch size, value len, hid dim]
+
+        Q = Q.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)
+        K = K.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)
+        V = V.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)
+
+        # Q = [batch size, n heads, query len, head dim]
+        # K = [batch size, n heads, key len, head dim]
+        # V = [batch size, n heads, value len, head dim]
+
+        energy = torch.matmul(Q, K.permute(0, 1, 3, 2)) / self.scale
+
+        # energy = [batch size, n heads, query len, key len]
+
+        if mask is not None:
+            energy = energy.masked_fill(mask == 0, -1e10)
+
+        attention = torch.softmax(energy, dim=-1)
+
+        # attention = [batch size, n heads, query len, key len]
+
+        x = torch.matmul(self.dropout(attention), V)
+
+        # x = [batch size, n heads, query len, head dim]
+
+        x = x.permute(0, 2, 1, 3).contiguous()
+
+        # x = [batch size, query len, n heads, head dim]
+
+        x = x.view(batch_size, -1, self.hid_dim)
+
+        # x = [batch size, query len, hid dim]
+
+        x = self.fc_o(x)
+
+        # x = [batch size, query len, hid dim]
+
+        return x, attention
+
+
+"""### Position-wise Feedforward Layer
+
+The other main block inside the encoder layer is the *position-wise feedforward layer* This is relatively simple compared to the multi-head attention layer. The input is transformed from `hid_dim` to `pf_dim`, where `pf_dim` is usually a lot larger than `hid_dim`. The original Transformer used a `hid_dim` of 512 and a `pf_dim` of 2048. The ReLU activation function and dropout are applied before it is transformed back into a `hid_dim` representation. 
+
+Why is this used? Unfortunately, it is never explained in the paper.
+
+BERT uses the [GELU](https://arxiv.org/abs/1606.08415) activation function, which can be used by simply switching `torch.relu` for `F.gelu`. Why did they use GELU? Again, it is never explained.
+"""
+
+
+class PositionwiseFeedforwardLayer(nn.Module):
+    def __init__(self, hid_dim, pf_dim, dropout):
+        super().__init__()
+
+        self.fc_1 = nn.Linear(hid_dim, pf_dim)
+        self.fc_2 = nn.Linear(pf_dim, hid_dim)
+
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        # x = [batch size, seq len, hid dim]
+
+        x = self.dropout(torch.relu(self.fc_1(x)))
+
+        # x = [batch size, seq len, pf dim]
+
+        x = self.fc_2(x)
+
+        # x = [batch size, seq len, hid dim]
+
+        return x
+
+
+"""### Decoder
+
+The objective of the decoder is to take the encoded representation of the source sentence, $Z$, and convert it into predicted tokens in the target sentence, $\hat{Y}$. We then compare $\hat{Y}$ with the actual tokens in the target sentence, $Y$, to calculate our loss, which will be used to calculate the gradients of our parameters and then use our optimizer to update our weights in order to improve our predictions. 
+
+![](https://github.com/bentrevett/pytorch-seq2seq/blob/master/assets/transformer-decoder.png?raw=1)
+
+The decoder is similar to encoder, however it now has two multi-head attention layers. A *masked multi-head attention layer* over the target sequence, and a multi-head attention layer which uses the decoder representation as the query and the encoder representation as the key and value.
+
+The decoder uses positional embeddings and combines - via an elementwise sum - them with the scaled embedded target tokens, followed by dropout. Again, our positional encodings have a "vocabulary" of 100, which means they can accept sequences up to 100 tokens long. This can be increased if desired.
+
+The combined embeddings are then passed through the $N$ decoder layers, along with the encoded source, `enc_src`, and the source and target masks. Note that the number of layers in the encoder does not have to be equal to the number of layers in the decoder, even though they are both denoted by $N$.
+
+The decoder representation after the $N^{th}$ layer is then passed through a linear layer, `fc_out`. In PyTorch, the softmax operation is contained within our loss function, so we do not explicitly need to use a softmax layer here.
+
+As well as using the source mask, as we did in the encoder to prevent our model attending to `<pad>` tokens, we also use a target mask. This will be explained further in the `Seq2Seq` model which encapsulates both the encoder and decoder, but the gist of it is that it performs a similar operation as the decoder padding in the convolutional sequence-to-sequence model. As we are processing all of the target tokens at once in parallel we need a method of stopping the decoder from "cheating" by simply "looking" at what the next token in the target sequence is and outputting it. 
+
+Our decoder layer also outputs the normalized attention values so we can later plot them to see what our model is actually paying attention to.
+"""
+
+
+class Decoder(nn.Module):
+    def __init__(self,
+                 output_dim,
+                 hid_dim,
+                 n_layers,
+                 n_heads,
+                 pf_dim,
+                 dropout,
+                 device,
+                 max_length=100):
+        super().__init__()
+
+        self.device = device
+
+        self.tok_embedding = nn.Embedding(output_dim, hid_dim)
+        self.pos_embedding = nn.Embedding(max_length, hid_dim)
+
+        self.layers = nn.ModuleList([DecoderLayer(hid_dim,
+                                                  n_heads,
+                                                  pf_dim,
+                                                  dropout,
+                                                  device)
+                                     for _ in range(n_layers)])
+
+        self.fc_out = nn.Linear(hid_dim, output_dim)
+
+        self.dropout = nn.Dropout(dropout)
+
+        self.scale = torch.sqrt(torch.FloatTensor([hid_dim])).to(device)
+
+    def forward(self, trg, enc_src, trg_mask, src_mask):
+        # trg = [batch size, trg len]
+        # enc_src = [batch size, src len, hid dim]
+        # trg_mask = [batch size, 1, trg len, trg len]
+        # src_mask = [batch size, 1, 1, src len]
+
+        batch_size = trg.shape[0]
+        trg_len = trg.shape[1]
+
+        pos = torch.arange(0, trg_len).unsqueeze(0).repeat(batch_size, 1).to(self.device)
+
+        # pos = [batch size, trg len]
+
+        trg = self.dropout((self.tok_embedding(trg) * self.scale) + self.pos_embedding(pos))
+
+        # trg = [batch size, trg len, hid dim]
+
+        for layer in self.layers:
+            trg, attention = layer(trg, enc_src, trg_mask, src_mask)
+
+        # trg = [batch size, trg len, hid dim]
+        # attention = [batch size, n heads, trg len, src len]
+
+        output = self.fc_out(trg)
+
+        # output = [batch size, trg len, output dim]
+
+        return output, attention
+
+
+"""### Decoder Layer
+
+As mentioned previously, the decoder layer is similar to the encoder layer except that it now has two multi-head attention layers, `self_attention` and `encoder_attention`. 
+
+The first performs self-attention, as in the encoder, by using the decoder representation so far as the query, key and value. This is followed by dropout, residual connection and layer normalization. This `self_attention` layer uses the target sequence mask, `trg_mask`, in order to prevent the decoder from "cheating" by paying attention to tokens that are "ahead" of the one it is currently processing as it processes all tokens in the target sentence in parallel.
+
+The second is how we actually feed the encoded source sentence, `enc_src`, into our decoder. In this multi-head attention layer the queries are the decoder representations and the keys and values are the encoder representations. Here, the source mask, `src_mask` is used to prevent the multi-head attention layer from attending to `<pad>` tokens within the source sentence. This is then followed by the dropout, residual connection and layer normalization layers. 
+
+Finally, we pass this through the position-wise feedforward layer and yet another sequence of dropout, residual connection and layer normalization.
+
+The decoder layer isn't introducing any new concepts, just using the same set of layers as the encoder in a slightly different way.
+"""
+
+
+class DecoderLayer(nn.Module):
+    def __init__(self,
+                 hid_dim,
+                 n_heads,
+                 pf_dim,
+                 dropout,
+                 device):
+        super().__init__()
+
+        self.self_attn_layer_norm = nn.LayerNorm(hid_dim)
+        self.enc_attn_layer_norm = nn.LayerNorm(hid_dim)
+        self.ff_layer_norm = nn.LayerNorm(hid_dim)
+        self.self_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)
+        self.encoder_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)
+        self.positionwise_feedforward = PositionwiseFeedforwardLayer(hid_dim,
+                                                                     pf_dim,
+                                                                     dropout)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, trg, enc_src, trg_mask, src_mask):
+        # trg = [batch size, trg len, hid dim]
+        # enc_src = [batch size, src len, hid dim]
+        # trg_mask = [batch size, 1, trg len, trg len]
+        # src_mask = [batch size, 1, 1, src len]
+
+        # self attention
+        _trg, _ = self.self_attention(trg, trg, trg, trg_mask)
+
+        # dropout, residual connection and layer norm
+        trg = self.self_attn_layer_norm(trg + self.dropout(_trg))
+
+        # trg = [batch size, trg len, hid dim]
+
+        # encoder attention
+        _trg, attention = self.encoder_attention(trg, enc_src, enc_src, src_mask)
+
+        # dropout, residual connection and layer norm
+        trg = self.enc_attn_layer_norm(trg + self.dropout(_trg))
+
+        # trg = [batch size, trg len, hid dim]
+
+        # positionwise feedforward
+        _trg = self.positionwise_feedforward(trg)
+
+        # dropout, residual and layer norm
+        trg = self.ff_layer_norm(trg + self.dropout(_trg))
+
+        # trg = [batch size, trg len, hid dim]
+        # attention = [batch size, n heads, trg len, src len]
+
+        return trg, attention
+
+
+"""### Seq2Seq
+
+Finally, we have the `Seq2Seq` module which encapsulates the encoder and decoder, as well as handling the creation of the masks.
+
+The source mask is created by checking where the source sequence is not equal to a `<pad>` token. It is 1 where the token is not a `<pad>` token and 0 when it is. It is then unsqueezed so it can be correctly broadcast when applying the mask to the `energy`, which of shape **_[batch size, n heads, seq len, seq len]_**.
+
+The target mask is slightly more complicated. First, we create a mask for the `<pad>` tokens, as we did for the source mask. Next, we create a "subsequent" mask, `trg_sub_mask`, using `torch.tril`. This creates a diagonal matrix where the elements above the diagonal will be zero and the elements below the diagonal will be set to whatever the input tensor is. In this case, the input tensor will be a tensor filled with ones. So this means our `trg_sub_mask` will look something like this (for a target with 5 tokens):
+
+$$\begin{matrix}
+1 & 0 & 0 & 0 & 0\\
+1 & 1 & 0 & 0 & 0\\
+1 & 1 & 1 & 0 & 0\\
+1 & 1 & 1 & 1 & 0\\
+1 & 1 & 1 & 1 & 1\\
+\end{matrix}$$
+
+This shows what each target token (row) is allowed to look at (column). The first target token has a mask of **_[1, 0, 0, 0, 0]_** which means it can only look at the first target token. The second target token has a mask of **_[1, 1, 0, 0, 0]_** which it means it can look at both the first and second target tokens. 
+
+The "subsequent" mask is then logically anded with the padding mask, this combines the two masks ensuring both the subsequent tokens and the padding tokens cannot be attended to. For example if the last two tokens were `<pad>` tokens the mask would look like:
+
+$$\begin{matrix}
+1 & 0 & 0 & 0 & 0\\
+1 & 1 & 0 & 0 & 0\\
+1 & 1 & 1 & 0 & 0\\
+1 & 1 & 1 & 0 & 0\\
+1 & 1 & 1 & 0 & 0\\
+\end{matrix}$$
+
+After the masks are created, they used with the encoder and decoder along with the source and target sentences to get our predicted target sentence, `output`, along with the decoder's attention over the source sequence.
+"""
+
+
+class Seq2Seq(nn.Module):
+    def __init__(self,
+                 encoder,
+                 decoder,
+                 src_pad_idx,
+                 trg_pad_idx,
+                 device):
+        super().__init__()
+
+        self.encoder = encoder
+        self.decoder = decoder
+        self.src_pad_idx = src_pad_idx
+        self.trg_pad_idx = trg_pad_idx
+        self.device = device
+
+    def make_src_mask(self, src):
+        # src = [batch size, src len]
+
+        src_mask = (src != self.src_pad_idx).unsqueeze(1).unsqueeze(2)
+
+        # src_mask = [batch size, 1, 1, src len]
+
+        return src_mask
+
+    def make_trg_mask(self, trg):
+        # trg = [batch size, trg len]
+
+        trg_pad_mask = (trg != self.trg_pad_idx).unsqueeze(1).unsqueeze(2)
+
+        # trg_pad_mask = [batch size, 1, 1, trg len]
+
+        trg_len = trg.shape[1]
+
+        trg_sub_mask = torch.tril(torch.ones((trg_len, trg_len), device=self.device)).bool()
+
+        # trg_sub_mask = [trg len, trg len]
+
+        trg_mask = trg_pad_mask & trg_sub_mask
+
+        # trg_mask = [batch size, 1, trg len, trg len]
+
+        return trg_mask
+
+    def forward(self, src, trg):
+        # src = [batch size, src len]
+        # trg = [batch size, trg len]
+
+        src_mask = self.make_src_mask(src)
+        trg_mask = self.make_trg_mask(trg)
+
+        # src_mask = [batch size, 1, 1, src len]
+        # trg_mask = [batch size, 1, trg len, trg len]
+
+        enc_src = self.encoder(src, src_mask)
+
+        # enc_src = [batch size, src len, hid dim]
+
+        output, attention = self.decoder(trg, enc_src, trg_mask, src_mask)
+
+        # output = [batch size, trg len, output dim]
+        # attention = [batch size, n heads, trg len, src len]
+
+        return output, attention
+
+
+"""## Training the Seq2Seq Model
+
+We can now define our encoder and decoders. This model is significantly smaller than Transformers used in research today, but is able to be run on a single GPU quickly.
+"""
+
+INPUT_DIM = len(SRC.vocab)
+OUTPUT_DIM = len(TRG.vocab)
+HID_DIM = 256
+ENC_LAYERS = 3
+DEC_LAYERS = 3
+ENC_HEADS = 8
+DEC_HEADS = 8
+ENC_PF_DIM = 512
+DEC_PF_DIM = 512
+ENC_DROPOUT = 0.1
+DEC_DROPOUT = 0.1
+
+enc = Encoder(INPUT_DIM,
+              HID_DIM,
+              ENC_LAYERS,
+              ENC_HEADS,
+              ENC_PF_DIM,
+              ENC_DROPOUT,
+              device)
+
+dec = Decoder(OUTPUT_DIM,
+              HID_DIM,
+              DEC_LAYERS,
+              DEC_HEADS,
+              DEC_PF_DIM,
+              DEC_DROPOUT,
+              device)
+
+"""Then, use them to define our whole sequence-to-sequence encapsulating model."""
+
+SRC_PAD_IDX = SRC.vocab.stoi[SRC.pad_token]
+TRG_PAD_IDX = TRG.vocab.stoi[TRG.pad_token]
+
+model = Seq2Seq(enc, dec, SRC_PAD_IDX, TRG_PAD_IDX, device).to(device)
+
+"""We can check the number of parameters, noticing it is significantly less than the 37M for the convolutional sequence-to-sequence model."""
+
+
+def count_parameters(model):
+    return sum(p.numel() for p in model.parameters() if p.requires_grad)
+
+
+print(f'The model has {count_parameters(model):,} trainable parameters')
+
+"""The paper does not mention which weight initialization scheme was used, however Xavier uniform seems to be common amongst Transformer models, so we use it here."""
+
+
+def initialize_weights(m):
+    if hasattr(m, 'weight') and m.weight.dim() > 1:
+        nn.init.xavier_uniform_(m.weight.data)
+
+
+model.apply(initialize_weights);
+
+"""The optimizer used in the original Transformer paper uses Adam with a learning rate that has a "warm-up" and then a "cool-down" period. BERT and other Transformer models use Adam with a fixed learning rate, so we will implement that. Check [this](http://nlp.seas.harvard.edu/2018/04/03/attention.html#optimizer) link for more details about the original Transformer's learning rate schedule.
+
+Note that the learning rate needs to be lower than the default used by Adam or else learning is unstable.
+"""
+
+LEARNING_RATE = 0.0005
+
+optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)
+
+"""Next, we define our loss function, making sure to ignore losses calculated over `<pad>` tokens."""
+
+criterion = nn.CrossEntropyLoss(ignore_index=TRG_PAD_IDX)
+
+"""Then, we'll define our training loop. This is the exact same as the one used in the previous tutorial.
+
+As we want our model to predict the `<eos>` token but not have it be an input into our model we simply slice the `<eos>` token off the end of the sequence. Thus:
+
+$$\begin{align*}
+\text{trg} &= [sos, x_1, x_2, x_3, eos]\\
+\text{trg[:-1]} &= [sos, x_1, x_2, x_3]
+\end{align*}$$
+
+$x_i$ denotes actual target sequence element. We then feed this into the model to get a predicted sequence that should hopefully predict the `<eos>` token:
+
+$$\begin{align*}
+\text{output} &= [y_1, y_2, y_3, eos]
+\end{align*}$$
+
+$y_i$ denotes predicted target sequence element. We then calculate our loss using the original `trg` tensor with the `<sos>` token sliced off the front, leaving the `<eos>` token:
+
+$$\begin{align*}
+\text{output} &= [y_1, y_2, y_3, eos]\\
+\text{trg[1:]} &= [x_1, x_2, x_3, eos]
+\end{align*}$$
+
+We then calculate our losses and update our parameters as is standard.
+"""
+
+
+def train(model, iterator, optimizer, criterion, clip):
+    model.train()
+
+    epoch_loss = 0
+
+    for i, batch in enumerate(iterator):
+        src = batch.src
+        trg = batch.trg
+
+        optimizer.zero_grad()
+
+        output, _ = model(src, trg[:, :-1])
+
+        # output = [batch size, trg len - 1, output dim]
+        # trg = [batch size, trg len]
+
+        output_dim = output.shape[-1]
+
+        output = output.contiguous().view(-1, output_dim)
+        trg = trg[:, 1:].contiguous().view(-1)
+
+        # output = [batch size * trg len - 1, output dim]
+        # trg = [batch size * trg len - 1]
+
+        loss = criterion(output, trg)
+
+        loss.backward()
+
+        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)
+
+        optimizer.step()
+
+        epoch_loss += loss.item()
+
+    return epoch_loss / len(iterator)
+
+
+"""The evaluation loop is the same as the training loop, just without the gradient calculations and parameter updates."""
+
+
+def evaluate(model, iterator, criterion):
+    model.eval()
+
+    epoch_loss = 0
+
+    with torch.no_grad():
+        for i, batch in enumerate(iterator):
+            src = batch.src
+            trg = batch.trg
+
+            output, _ = model(src, trg[:, :-1])
+
+            # output = [batch size, trg len - 1, output dim]
+            # trg = [batch size, trg len]
+
+            output_dim = output.shape[-1]
+
+            output = output.contiguous().view(-1, output_dim)
+            trg = trg[:, 1:].contiguous().view(-1)
+
+            # output = [batch size * trg len - 1, output dim]
+            # trg = [batch size * trg len - 1]
+
+            loss = criterion(output, trg)
+
+            epoch_loss += loss.item()
+
+    return epoch_loss / len(iterator)
+
+
+"""We then define a small function that we can use to tell us how long an epoch takes."""
+
+
+def epoch_time(start_time, end_time):
+    elapsed_time = end_time - start_time
+    elapsed_mins = int(elapsed_time / 60)
+    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))
+    return elapsed_mins, elapsed_secs
+
+
+"""Finally, we train our actual model. This model is almost 3x faster than the convolutional sequence-to-sequence model and also achieves a lower validation perplexity!"""
+
+N_EPOCHS = 10
+CLIP = 1
+
+best_valid_loss = float('inf')
+
+for epoch in range(N_EPOCHS):
+
+    start_time = time.time()
+
+    train_loss = train(model, train_iterator, optimizer, criterion, CLIP)
+    valid_loss = evaluate(model, valid_iterator, criterion)
+
+    end_time = time.time()
+
+    epoch_mins, epoch_secs = epoch_time(start_time, end_time)
+
+    if valid_loss < best_valid_loss:
+        best_valid_loss = valid_loss
+        torch.save(model.state_dict(), 'tut6-model.pt')
+
+    print(f'Epoch: {epoch + 1:02} | Time: {epoch_mins}m {epoch_secs}s')
+    print(f'\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')
+    print(f'\t Val. Loss: {valid_loss:.3f} |  Val. PPL: {math.exp(valid_loss):7.3f}')
+
+"""We load our "best" parameters and manage to achieve a better test perplexity than all previous models."""
+
+model.load_state_dict(torch.load('tut6-model.pt'))
+
+test_loss = evaluate(model, test_iterator, criterion)
+
+print(f'| Test Loss: {test_loss:.3f} | Test PPL: {math.exp(test_loss):7.3f} |')
+
+"""## Inference
+
+Now we can can translations from our model with the `translate_sentence` function below.
+
+The steps taken are:
+- tokenize the source sentence if it has not been tokenized (is a string)
+- append the `<sos>` and `<eos>` tokens
+- numericalize the source sentence
+- convert it to a tensor and add a batch dimension
+- create the source sentence mask
+- feed the source sentence and mask into the encoder
+- create a list to hold the output sentence, initialized with an `<sos>` token
+- while we have not hit a maximum length
+  - convert the current output sentence prediction into a tensor with a batch dimension
+  - create a target sentence mask
+  - place the current output, encoder output and both masks into the decoder
+  - get next output token prediction from decoder along with attention
+  - add prediction to current output sentence prediction
+  - break if the prediction was an `<eos>` token
+- convert the output sentence from indexes to tokens
+- return the output sentence (with the `<sos>` token removed) and the attention from the last layer
+"""
+
+
+def translate_sentence(sentence, src_field, trg_field, model, device, max_len=50):
+    model.eval()
+
+    if isinstance(sentence, str):
+        nlp = spacy.load('de_core_news_sm')
+        tokens = [token.text.lower() for token in nlp(sentence)]
+    else:
+        tokens = [token.lower() for token in sentence]
+
+    tokens = [src_field.init_token] + tokens + [src_field.eos_token]
+
+    src_indexes = [src_field.vocab.stoi[token] for token in tokens]
+
+    src_tensor = torch.LongTensor(src_indexes).unsqueeze(0).to(device)
+
+    src_mask = model.make_src_mask(src_tensor)
+
+    with torch.no_grad():
+        enc_src = model.encoder(src_tensor, src_mask)
+
+    trg_indexes = [trg_field.vocab.stoi[trg_field.init_token]]
+
+    for i in range(max_len):
+
+        trg_tensor = torch.LongTensor(trg_indexes).unsqueeze(0).to(device)
+
+        trg_mask = model.make_trg_mask(trg_tensor)
+
+        with torch.no_grad():
+            output, attention = model.decoder(trg_tensor, enc_src, trg_mask, src_mask)
+
+        pred_token = output.argmax(2)[:, -1].item()
+
+        trg_indexes.append(pred_token)
+
+        if pred_token == trg_field.vocab.stoi[trg_field.eos_token]:
+            break
+
+    trg_tokens = [trg_field.vocab.itos[i] for i in trg_indexes]
+
+    return trg_tokens[1:], attention
+
+
+"""We'll now define a function that displays the attention over the source sentence for each step of the decoding. As this model has 8 heads our model we can view the attention for each of the heads."""
+
+
+def display_attention(sentence, translation, attention, n_heads=8, n_rows=4, n_cols=2):
+    assert n_rows * n_cols == n_heads
+
+    fig = plt.figure(figsize=(15, 25))
+
+    for i in range(n_heads):
+        ax = fig.add_subplot(n_rows, n_cols, i + 1)
+
+        _attention = attention.squeeze(0)[i].cpu().detach().numpy()
+
+        cax = ax.matshow(_attention, cmap='bone')
+
+        ax.tick_params(labelsize=12)
+        ax.set_xticklabels([''] + ['<sos>'] + [t.lower() for t in sentence] + ['<eos>'],
+                           rotation=45)
+        ax.set_yticklabels([''] + translation)
+
+        ax.xaxis.set_major_locator(ticker.MultipleLocator(1))
+        ax.yaxis.set_major_locator(ticker.MultipleLocator(1))
+
+    plt.show()
+    plt.close()
+
+
+"""First, we'll get an example from the training set."""
+
+example_idx = 8
+
+src = vars(train_data.examples[example_idx])['src']
+trg = vars(train_data.examples[example_idx])['trg']
+
+print(f'src = {src}')
+print(f'trg = {trg}')
+
+"""Our translation looks pretty good, although our model changes *is walking by* to *walks by*. The meaning is still the same."""
+
+translation, attention = translate_sentence(src, SRC, TRG, model, device)
+
+print(f'predicted trg = {translation}')
+
+"""We can see the attention from each head below. Each is certainly different, but it's difficult (perhaps impossible) to reason about what head has actually learned to pay attention to. Some heads pay full attention to "eine" when translating "a", some don't at all, and some do a little. They all seem to follow the similar "downward staircase" pattern and the attention when outputting the last two tokens is equally spread over the final two tokens in the input sentence."""
+
+display_attention(src, translation, attention)
+
+"""Next, let's get an example the model has not been trained on from the validation set."""
+
+example_idx = 6
+
+src = vars(valid_data.examples[example_idx])['src']
+trg = vars(valid_data.examples[example_idx])['trg']
+
+print(f'src = {src}')
+print(f'trg = {trg}')
+
+"""The model translates it by switching *is running* to just *runs*, but it is an acceptable swap."""
+
+translation, attention = translate_sentence(src, SRC, TRG, model, device)
+
+print(f'predicted trg = {translation}')
+
+"""Again, some heads pay full attention to "ein" whilst some pay no attention to it. Again, most of the heads seem to spread their attention over both the period and `<eos>` tokens in the source sentence when outputting the period and `<eos>` sentence in the predicted target sentence, though some seem to pay attention to tokens from near the start of the sentence."""
+
+display_attention(src, translation, attention)
+
+"""Finally, we'll look at an example from the test data."""
+
+example_idx = 10
+
+src = vars(test_data.examples[example_idx])['src']
+trg = vars(test_data.examples[example_idx])['trg']
+
+print(f'src = {src}')
+print(f'trg = {trg}')
+
+"""A perfect translation!"""
+
+translation, attention = translate_sentence(src, SRC, TRG, model, device)
+
+print(f'predicted trg = {translation}')
+
+display_attention(src, translation, attention)
+
+"""## BLEU
+
+Finally we calculate the BLEU score for the Transformer.
+"""
+
+from torchtext.data.metrics import bleu_score
+
+
+def calculate_bleu(data, src_field, trg_field, model, device, max_len=50):
+    trgs = []
+    pred_trgs = []
+
+    for datum in data:
+        src = vars(datum)['src']
+        trg = vars(datum)['trg']
+
+        pred_trg, _ = translate_sentence(src, src_field, trg_field, model, device, max_len)
+
+        # cut off <eos> token
+        pred_trg = pred_trg[:-1]
+
+        pred_trgs.append(pred_trg)
+        trgs.append([trg])
+
+    return bleu_score(pred_trgs, trgs)
+
+
+"""We get a BLEU score of 36.52, which beats the ~34 of the convolutional sequence-to-sequence model and ~28 of the attention based RNN model. All this whilst having the least amount of parameters and the fastest training time!"""
+
+bleu_score = calculate_bleu(test_data, SRC, TRG, model, device)
+
+print(f'BLEU score = {bleu_score * 100:.2f}')
+
+"""Congratulations for finishing these tutorials! I hope you've found them useful.
+
+If you find any mistakes or want to ask any questions about any of the code or explanations used, feel free to submit a GitHub issue and I will try to correct it ASAP.
+
+## Appendix
+
+The `calculate_bleu` function above is unoptimized. Below is a significantly faster, vectorized version of it that should be used if needed. Credit for the implementation goes to [@azadyasar](https://github.com/azadyasar).
+"""
+
+
+def translate_sentence_vectorized(src_tensor, src_field, trg_field, model, device, max_len=50):
+    assert isinstance(src_tensor, torch.Tensor)
+
+    model.eval()
+    src_mask = model.make_src_mask(src_tensor)
+
+    with torch.no_grad():
+        enc_src = model.encoder(src_tensor, src_mask)
+    # enc_src = [batch_sz, src_len, hid_dim]
+
+    trg_indexes = [[trg_field.vocab.stoi[trg_field.init_token]] for _ in range(len(src_tensor))]
+    # Even though some examples might have been completed by producing a <eos> token
+    # we still need to feed them through the model because other are not yet finished
+    # and all examples act as a batch. Once every single sentence prediction encounters
+    # <eos> token, then we can stop predicting.
+    translations_done = [0] * len(src_tensor)
+    for i in range(max_len):
+        trg_tensor = torch.LongTensor(trg_indexes).to(device)
+        trg_mask = model.make_trg_mask(trg_tensor)
+        with torch.no_grad():
+            output, attention = model.decoder(trg_tensor, enc_src, trg_mask, src_mask)
+        pred_tokens = output.argmax(2)[:, -1]
+        for i, pred_token_i in enumerate(pred_tokens):
+            trg_indexes[i].append(pred_token_i)
+            if pred_token_i == trg_field.vocab.stoi[trg_field.eos_token]:
+                translations_done[i] = 1
+        if all(translations_done):
+            break
+
+    # Iterate through each predicted example one by one;
+    # Cut-off the portion including the after the <eos> token
+    pred_sentences = []
+    for trg_sentence in trg_indexes:
+        pred_sentence = []
+        for i in range(1, len(trg_sentence)):
+            if trg_sentence[i] == trg_field.vocab.stoi[trg_field.eos_token]:
+                break
+            pred_sentence.append(trg_field.vocab.itos[trg_sentence[i]])
+        pred_sentences.append(pred_sentence)
+
+    return pred_sentences, attention
+
+
+from torchtext.data.metrics import bleu_score
+
+
+def calculate_bleu_alt(iterator, src_field, trg_field, model, device, max_len=50):
+    trgs = []
+    pred_trgs = []
+    with torch.no_grad():
+        for batch in iterator:
+            src = batch.src
+            trg = batch.trg
+            _trgs = []
+            for sentence in trg:
+                tmp = []
+                # Start from the first token which skips the <start> token
+                for i in sentence[1:]:
+                    # Targets are padded. So stop appending as soon as a padding or eos token is encountered
+                    if i == trg_field.vocab.stoi[trg_field.eos_token] or i == trg_field.vocab.stoi[trg_field.pad_token]:
+                        break
+                    tmp.append(trg_field.vocab.itos[i])
+                _trgs.append([tmp])
+            trgs += _trgs
+            pred_trg, _ = translate_sentence_vectorized(src, src_field, trg_field, model, device)
+            pred_trgs += pred_trg
+    return pred_trgs, trgs, bleu_score(pred_trgs, trgs)
\ No newline at end of file
diff --git a/scripts/train.py b/scripts/train.py
new file mode 100644
index 0000000..a124309
--- /dev/null
+++ b/scripts/train.py
@@ -0,0 +1,189 @@
+import datetime
+import pprint
+import random
+import time
+import sys
+import numpy as np
+
+from torch.utils.data import DataLoader
+
+[sys.path.append(i) for i in ['.', '..']]
+
+from model import vocab
+from model.seq2seq_net import Seq2SeqNet
+from train_eval.train_seq2seq import train_iter_seq2seq
+from utils.average_meter import AverageMeter
+from utils.vocab_utils import build_vocab
+
+from config.parse_args import parse_args
+
+from torch import optim
+
+from data_loader.lmdb_data_loader import *
+import utils.train_utils
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+
+def init_model(args, lang_model, pose_dim, _device):
+    n_frames = args.n_poses
+    generator = Seq2SeqNet(args, pose_dim, n_frames, lang_model.n_words, args.wordembed_dim,
+                           lang_model.word_embedding_weights).to(_device)
+    loss_fn = torch.nn.MSELoss()
+
+    return generator, loss_fn
+
+
+def train_epochs(args, train_data_loader, test_data_loader, lang_model, pose_dim, trial_id=None):
+    start = time.time()
+    loss_meters = [AverageMeter('loss'), AverageMeter('var_loss')]
+
+    # interval params
+    print_interval = int(len(train_data_loader) / 5)
+    save_model_epoch_interval = 20
+
+    # init model
+    generator, loss_fn = init_model(args, lang_model, pose_dim, device)
+
+    # define optimizers
+    gen_optimizer = optim.Adam(generator.parameters(), lr=args.learning_rate, betas=(0.5, 0.999))
+
+    # training
+    global_iter = 0
+    for epoch in range(1, args.epochs+1):
+        # evaluate the test set
+        val_metrics = evaluate_testset(test_data_loader, generator, loss_fn, args)
+
+        # save model
+        if epoch % save_model_epoch_interval == 0 and epoch > 0:
+            try:  # multi gpu
+                gen_state_dict = generator.module.state_dict()
+            except AttributeError:  # single gpu
+                gen_state_dict = generator.state_dict()
+
+            save_name = '{}/{}_checkpoint_{:03d}.bin'.format(args.model_save_path, args.name, epoch)
+
+            utils.train_utils.save_checkpoint({
+                'args': args, 'epoch': epoch, 'lang_model': lang_model,
+                'pose_dim': pose_dim, 'gen_dict': gen_state_dict
+            }, save_name)
+
+        # train iter
+        iter_start_time = time.time()
+        for iter_idx, data in enumerate(train_data_loader, 0):
+            global_iter += 1
+            in_text, text_lengths, target_vec, in_audio, aux_info = data
+            batch_size = target_vec.size(0)
+
+            in_text = in_text.to(device)
+            # in_audio = in_audio.to(device)
+            target_vec = target_vec.to(device)
+
+            # train
+            loss = train_iter_seq2seq(args, epoch, in_text, text_lengths, target_vec, generator, gen_optimizer)
+
+            # loss values
+            for loss_meter in loss_meters:
+                name = loss_meter.name
+                if name in loss:
+                    loss_meter.update(loss[name], batch_size)
+
+            # print training status
+            if (iter_idx + 1) % print_interval == 0:
+                print_summary = 'EP {} ({:3d}) | {:>8s}, {:.0f} samples/s | '.format(
+                    epoch, iter_idx + 1, utils.train_utils.time_since(start),
+                    batch_size / (time.time() - iter_start_time))
+                for loss_meter in loss_meters:
+                    if loss_meter.count > 0:
+                        print_summary += '{}: {:.3f}, '.format(loss_meter.name, loss_meter.avg)
+                        loss_meter.reset()
+                logging.info(print_summary)
+
+            iter_start_time = time.time()
+
+
+def evaluate_testset(test_data_loader, generator, loss_fn, args):
+    # to evaluation mode
+    generator.train(False)
+
+    losses = AverageMeter('loss')
+    start = time.time()
+
+    with torch.no_grad():
+        for iter_idx, data in enumerate(test_data_loader, 0):
+            in_text, text_lengths, target_vec, in_audio, aux_info = data
+            batch_size = target_vec.size(0)
+
+            in_text = in_text.to(device)
+            target = target_vec.to(device)
+
+            out_poses = generator(in_text, text_lengths, target, None)
+            loss = loss_fn(out_poses, target)
+            losses.update(loss.item(), batch_size)
+
+    # back to training mode
+    generator.train(True)
+
+    # print
+    elapsed_time = time.time() - start
+    logging.info('[VAL] loss: {:.3f} / {:.1f}s'.format(losses.avg, elapsed_time))
+
+    return losses.avg
+
+
+def main(config):
+    args = config['args']
+
+    trial_id = None
+
+    # random seed
+    if args.random_seed >= 0:
+        torch.manual_seed(args.random_seed)
+        torch.cuda.manual_seed_all(args.random_seed)
+        np.random.seed(args.random_seed)
+        random.seed(args.random_seed)
+        os.environ['PYTHONHASHSEED'] = str(args.random_seed)
+
+    # set logger
+    utils.train_utils.set_logger(args.model_save_path, os.path.basename(__file__).replace('.py', '.log'))
+
+    logging.info("PyTorch version: {}".format(torch.__version__))
+    logging.info("CUDA version: {}".format(torch.version.cuda))
+    logging.info(pprint.pformat(vars(args)))
+
+    # dataset
+    train_dataset = TrinityDataset(args, args.train_data_path[0],
+                                   n_poses=args.n_poses,
+                                   subdivision_stride=args.subdivision_stride,
+                                   pose_resampling_fps=args.motion_resampling_framerate,
+                                   data_mean=args.data_mean, data_std=args.data_std)
+    train_loader = DataLoader(dataset=train_dataset, batch_size=args.batch_size,
+                              shuffle=True, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                              collate_fn=word_seq_collate_fn
+                              )
+
+    val_dataset = TrinityDataset(args, args.val_data_path[0],
+                                 n_poses=args.n_poses,
+                                 subdivision_stride=args.subdivision_stride,
+                                 pose_resampling_fps=args.motion_resampling_framerate,
+                                 data_mean=args.data_mean, data_std=args.data_std)
+    test_loader = DataLoader(dataset=val_dataset, batch_size=args.batch_size,
+                             shuffle=False, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                             collate_fn=word_seq_collate_fn
+                             )
+
+    # build vocab
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    lang_model = build_vocab('words', [train_dataset, val_dataset], vocab_cache_path, args.wordembed_path,
+                             args.wordembed_dim)
+    train_dataset.set_lang_model(lang_model)
+    val_dataset.set_lang_model(lang_model)
+
+    # train
+    train_epochs(args, train_loader, test_loader, lang_model,
+                 pose_dim=15*9, trial_id=trial_id)
+
+
+if __name__ == '__main__':
+    _args = parse_args()
+    main({'args': _args})
diff --git a/scripts/train_Autoencoder.py b/scripts/train_Autoencoder.py
new file mode 100644
index 0000000..a7b9c84
--- /dev/null
+++ b/scripts/train_Autoencoder.py
@@ -0,0 +1,568 @@
+import datetime
+import os
+import pprint
+import random
+import time
+import sys
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.signal import savgol_filter
+
+import seaborn as sns
+from sklearn.decomposition import PCA
+
+from torch.utils.data import DataLoader
+from tqdm import tqdm
+import torch
+[sys.path.append(i) for i in ['.', '..']]
+
+from model import vocab
+from model.seq2seq_net import Seq2SeqNet
+from model.Autoencoder_model import Autoencoder_seq2seq
+from train_eval.train_seq2seq import train_iter_seq2seq, train_iter_Autoencoder_seq2seq, train_iter_Autoencoder_ssl_seq2seq
+from utils.average_meter import AverageMeter
+from utils.vocab_utils import build_vocab
+
+from config.parse_args import parse_args
+
+from torch import optim
+
+from data_loader.lmdb_data_loader import *
+import utils.train_utils
+
+import utils
+from pymo.preprocessing import *
+from pymo.viz_tools import *
+from pymo.writers import *
+from utils.data_utils import SubtitleWrapper, normalize_string
+import joblib as jl
+
+from openTSNE import TSNE
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+debug = False
+
+def init_model(args, lang_model, pose_dim, _device):
+    n_frames = args.n_poses
+    generator = Autoencoder_seq2seq(args, pose_dim, n_frames).to(_device)
+    loss_fn = torch.nn.MSELoss()
+
+    return generator, loss_fn
+
+
+
+
+def train_epochs(args, train_data_loader, train_sim_dataset, test_data_loader, lang_model, pose_dim, trial_id=None):
+    start = time.time()
+    loss_meters = [AverageMeter('loss'), AverageMeter('var_loss')]
+
+    if args.use_derivitive=='True':
+        pass
+
+    # interval params
+    print_interval = int(len(train_data_loader))
+    save_model_epoch_interval = 20
+
+    # init model
+    generator, loss_fn = init_model(args, lang_model, pose_dim, device)
+    if args.autoencoder_freeze_encoder == 'True':
+        rnn = utils.train_utils.load_checkpoint_and_model(
+            args.autoencoder_checkpoint, device, 'autoencoder')
+        args_rnn, rnn, loss_fn, lang_model, out_dim = rnn
+        generator = rnn
+        generator.freez_encoder()
+        generator.autoencoder_fixed_weight = False
+        generator.autoencoder_conditioned = True
+        generator.decoder.decoder.autoencoder_conditioned = True
+        for param in generator.decoder.parameters():
+            param.requires_grad = True
+
+    # define optimizers
+    gen_optimizer = optim.Adam(generator.parameters(), lr=args.learning_rate, betas=(0.5, 0.999))
+
+    val_metrics = evaluate_testset(test_data_loader, generator, loss_fn, args)
+
+
+    # To record loss and evaluations metric:
+    val_metrics_list = []
+    loss_list = []
+
+
+    # training
+    global_iter = 0
+
+    for epoch in range(1, args.epochs+1):
+
+
+        # evaluate the test set
+        val_metrics = evaluate_testset(test_data_loader, generator, loss_fn, args)
+        val_metrics_list.append(val_metrics)
+        # save model
+        if epoch % save_model_epoch_interval == 0 and epoch > 0:
+            try:  # multi gpu
+                gen_state_dict = generator.module.state_dict()
+            except AttributeError:  # single gpu
+                gen_state_dict = generator.state_dict()
+
+            save_name = '{}/{}_checkpoint_{:03d}.bin'.format(args.model_save_path, args.name, epoch)
+
+            utils.train_utils.save_checkpoint({
+                'args': args,
+                'epoch': epoch,
+                'lang_model': lang_model,
+                'pose_dim': pose_dim,
+                'gen_dict': gen_state_dict,
+                'val_metrics_list': val_metrics_list,
+                'loss_list': loss_list
+
+            }, save_name)
+
+        # train iter
+        iter_start_time = time.time()
+        loss_epoch = AverageMeter('loss')
+        loss_epoch.reset()
+        # Todo: move to args
+        train_labeled = False
+        labeled_count = 20
+        for iter_idx, data in enumerate(train_data_loader, 0):
+            global_iter += 1
+            encoded_input, encoded_output = data
+            batch_size = encoded_output.size(0)
+
+            encoded_input = encoded_input.to(device)
+            encoded_output = encoded_output.to(device)
+
+            # train
+
+            if train_labeled:
+                stack_pairs1, stack_pairs2, stack_labels = train_sim_dataset.get_labeled_(labeled_count)
+
+                if debug:
+                    print("STAAAAACK", stack_pairs1.shape)
+                    print("STAAAAACK_LLL", stack_labels.shape)
+                stack_pairs1 = stack_pairs1.to(device)
+                stack_pairs2 = stack_pairs2.to(device)
+                stack_labels = stack_labels.to(device)
+
+
+                loss = train_iter_Autoencoder_ssl_seq2seq(args, epoch, encoded_input, encoded_output, generator,
+                                                      gen_optimizer,
+                                                          stack_pairs1, stack_pairs2, stack_labels)
+            else:
+                loss = train_iter_Autoencoder_seq2seq(args, epoch,
+                                                      encoded_input, encoded_output, generator, gen_optimizer)
+
+
+            loss_epoch.update(loss['loss'], batch_size)
+            # loss values
+            for loss_meter in loss_meters:
+                name = loss_meter.name
+                if name in loss:
+                    loss_meter.update(loss[name], batch_size)
+
+            # print training status
+            if (iter_idx + 1) % print_interval == 0:
+                print_summary = 'EP {} ({:3d}) | {:>8s}, {:.0f} samples/s | '.format(
+                    epoch, iter_idx + 1, utils.train_utils.time_since(start),
+                    batch_size / (time.time() - iter_start_time))
+                for loss_meter in loss_meters:
+                    if loss_meter.count > 0:
+                        print_summary += '{}: {:.3f}, '.format(loss_meter.name, loss_meter.avg)
+                        loss_meter.reset()
+                logging.info(print_summary)
+
+            iter_start_time = time.time()
+
+        loss_list.append(loss_epoch.avg)
+    plot_loss(args, all_eval_loss=val_metrics_list, all_train_loss=loss_list)
+
+
+def evaluate_testset(test_data_loader, generator, loss_fn, args):
+    # to evaluation mode
+    generator.train(False)
+
+    losses = AverageMeter('loss')
+    start = time.time()
+
+    with torch.no_grad():
+        for iter_idx, data in enumerate(test_data_loader, 0):
+            in_poses, target_poses = data
+            batch_size = in_poses.size(0)
+            if debug:
+                print("in_shape", in_poses.shape, "Target_poses", target_poses.shape)
+            in_poses = in_poses.to(device)
+            target_poses = target_poses.to(device)
+            if args.autoencoder_vae== 'True':
+                out_poses, latent_, mue, logvar = generator(in_poses, target_poses)
+            else:
+                # print(in_poses.shape,target_poses.shape)
+                # print(generator)
+                out_poses, latent_ = generator(in_poses, target_poses)
+
+            loss = loss_fn(out_poses, target_poses)
+            losses.update(loss.item(), batch_size)
+
+    # back to training mode
+    generator.train(True)
+
+    # print
+    elapsed_time = time.time() - start
+    logging.info('[VAL] loss: {:.3f} / {:.1f}s'.format(losses.avg, elapsed_time))
+
+    return losses.avg
+
+
+
+def plot_loss(args, all_eval_loss, all_train_loss):
+
+
+    # X = np.arange(136-3)
+    # fig = plt.figure()
+    # ax = fig.add_axes([0, 0, 1, 1])
+    # ax.bar(X + 0.00, all_train_loss[0], color='b', width=0.25)
+    # ax.bar(X + 0.25, all_eval_loss[1], color='g', width=0.25)
+    # plt.show()
+    # plotting the second plot
+    plt.plot(all_train_loss, label='Train loss')
+    plt.plot(all_eval_loss, label='Evaluation loss')
+
+    # Labeling the X-axis
+    plt.xlabel('Epoch number')
+    # Labeling the Y-axis
+    plt.ylabel('Loss Average')
+    # Give a title to the graph
+    plt.title('Training/Evaluation Loss based on epoch number')
+
+    # Show a legend on the plot
+    plt.legend()
+
+    plt.savefig(os.path.join(args.model_save_path, 'loss_plot.png'))
+    plt.show()
+
+def plot_embedding(args, rnn, epoch_num,
+                   train_res_recon_error,
+                   train_res_perplexity):
+
+
+    w = rnn.vq_layer._embedding.weight.cpu().detach().numpy()
+    # PCA
+    pca = PCA(n_components=20)
+    priciple_components = pca.fit(w)
+    normalized_data = priciple_components.transform(w)
+    # TSNE
+
+    MyTSNE = TSNE(
+        n_components=2,
+        perplexity=30,
+        metric='euclidean',
+        n_jobs=8,
+        verbose=True
+    )
+    X_embedded = MyTSNE.fit(normalized_data)
+
+    plt.figure(figsize=(16, 10))
+    # palette = sns.color_palette("bright", k_component)
+    # 2D
+
+    sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1],
+                    # hue=labels_,
+                    legend=False)
+    plt.title("Embedding visualization" + str(epoch_num))
+    address = (args.model_save_path) + '/plots/'
+    os.makedirs(address, exist_ok=True)
+    address += 'Embedding_Epoch(' + str(epoch_num) +').png'
+    plt.savefig(address)
+    plt.clf()
+    plt.show()
+
+
+    # Reconstruction loss and perplexity
+    print(train_res_perplexity)
+    try:
+        train_res_recon_error_smooth = savgol_filter(train_res_recon_error, 201, 7)
+        train_res_perplexity_smooth =  savgol_filter(train_res_perplexity, 201, 7)
+    except:
+        train_res_recon_error_smooth = train_res_recon_error #savgol_filter(train_res_recon_error, 201, 7)
+        train_res_perplexity_smooth = train_res_perplexity #[0] #savgol_filter(train_res_perplexity, 201, 7)
+
+    f = plt.figure(figsize=(16, 8))
+    ax = f.add_subplot(1, 2, 1)
+    ax.plot(train_res_recon_error_smooth)
+    ax.set_yscale('log')
+    ax.set_title('Smoothed NMSE.')
+    ax.set_xlabel('iteration')
+
+    ax = f.add_subplot(1, 2, 2)
+    ax.plot(train_res_perplexity_smooth)
+    ax.set_title('Smoothed Average codebook usage (perplexity).')
+    ax.set_xlabel('iteration')
+
+    address = (args.model_save_path) + '/plots/Error_Epoch(' + str(epoch_num) + ').png'
+
+    plt.savefig(address)
+    plt.clf()
+    plt.show()
+
+
+
+
+    pass
+
+
+
+
+def save_sim_graph_gesture(args, pairs, data_original):
+    save_result_path = args.model_save_path
+    sim_count = 0
+    dissim_count = 0
+    for item in tqdm(pairs):
+        if item[2] == 0:
+            dissim_count += 1
+            directory = str(save_result_path) + '/sim_graph/dis/' + str(dissim_count)
+            os.makedirs(directory)
+            make_bvh(directory, str(item[0]), data_original[item[0]])
+            make_bvh(directory, str(item[1]), data_original[item[1]])
+        else:
+            sim_count = sim_count + 1
+            directory = str(save_result_path) + '/sim_graph/sim/' + str(sim_count)
+            os.makedirs(directory)
+            make_bvh(directory, str(item[0]), data_original[item[0]])
+            make_bvh(directory, str(item[1]), data_original[item[1]])
+def make_bvh(save_path, filename_prefix, poses):
+    writer = BVHWriter()
+    pipeline = jl.load('../resource/data_pipe.sav')
+
+    # smoothing
+    n_poses = poses.shape[0]
+    out_poses = np.zeros((n_poses, poses.shape[1]))
+
+    for i in range(poses.shape[1]):
+        out_poses[:, i] = savgol_filter(poses[:, i], 15, 2)  # NOTE: smoothing on rotation matrices is not optimal
+
+    # rotation matrix to euler angles
+    out_poses = out_poses.reshape((out_poses.shape[0], -1, 9))
+    out_poses = out_poses.reshape((out_poses.shape[0], out_poses.shape[1], 3, 3))
+    out_euler = np.zeros((out_poses.shape[0], out_poses.shape[1] * 3))
+    for i in range(out_poses.shape[0]):  # frames
+        r = R.from_matrix(out_poses[i])
+        out_euler[i] = r.as_euler('ZXY', degrees=True).flatten()
+
+    bvh_data = pipeline.inverse_transform([out_euler])
+
+    out_bvh_path = os.path.join(save_path, filename_prefix + '_generated.bvh')
+    with open(out_bvh_path, 'w') as f:
+        writer.write(bvh_data[0], f)
+
+
+
+
+
+def main(config):
+    args = config['args']
+
+    trial_id = None
+
+    # random seed
+    if args.random_seed >= 0:
+        torch.manual_seed(args.random_seed)
+        torch.cuda.manual_seed_all(args.random_seed)
+        np.random.seed(args.random_seed)
+        random.seed(args.random_seed)
+        os.environ['PYTHONHASHSEED'] = str(args.random_seed)
+
+    # set logger
+    utils.train_utils.set_logger(args.model_save_path, os.path.basename(__file__).replace('.py', '.log'))
+
+    logging.info("PyTorch version: {}".format(torch.__version__))
+    logging.info("CUDA version: {}".format(torch.version.cuda))
+    logging.info(pprint.pformat(vars(args)))
+
+    # dataset
+    # Todo: change it backt to the real train
+    train_dataset = TrinityDataset_DAEed_Autoencoder(args, args.train_data_path[0],
+                                   n_poses=args.n_poses,
+                                   subdivision_stride=args.subdivision_stride,
+                                   pose_resampling_fps=args.motion_resampling_framerate,
+                                   data_mean=args.data_mean, data_std=args.data_std)
+
+    if args.use_similarity == 'True':
+        print()
+        train_dataset.creat_similarity_dataset(args.data_for_sim, args.similarity_labels)
+    #     Sanity check
+    #     save_sim_graph_gesture(args, train_dataset.pairwise_labels, train_dataset.data_original)
+    #     print("Sanity check saved!")
+
+
+    train_loader = DataLoader(dataset=train_dataset, batch_size=args.batch_size,
+                              shuffle=True, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                              # collate_fn=word_seq_collate_fn
+                              )
+
+
+    val_dataset = TrinityDataset_DAEed_Autoencoder(args, args.val_data_path[0],
+                                 n_poses=args.n_poses,
+                                 subdivision_stride=args.subdivision_stride,
+                                 pose_resampling_fps=args.motion_resampling_framerate,
+                                 data_mean=args.data_mean, data_std=args.data_std)
+
+    test_loader = DataLoader(dataset=val_dataset, batch_size=args.batch_size,
+                             shuffle=False, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                             # collate_fn=word_seq_collate_fn
+                             )
+
+    # build vocab
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    # lang_model = build_vocab('words', [train_dataset, val_dataset], vocab_cache_path, args.wordembed_path,
+    #                          args.wordembed_dim)
+    # train_dataset.set_lang_model(lang_model)
+    # val_dataset.set_lang_model(lang_model)
+
+    # train
+
+    # # Ablation study:
+    # # for autoencoder_denoising in [ 'False', 'True']:
+    # counter = 0
+    # ablation_folder = args.model_save_path
+    # for autoencoder_att in [ 'False', 'True']:
+    #     args.autoencoder_att = autoencoder_att
+    #     for autoencoder_fixed_weight in [ 'False', 'True']:
+    #         args.autoencoder_fixed_weight = autoencoder_fixed_weight
+    #         for autoencoder_conditioned in [ 'False', 'True']:
+    #             args.autoencoder_conditioned = autoencoder_conditioned
+    #             for autoencoder_vae in [ 'False', 'True']:
+    #                 args.autoencoder_vae = autoencoder_vae
+    #                 for autoencoder_vq in [ 'False']: #, 'True']:
+    #                     args.autoencoder_vq = autoencoder_vq
+    #                     for use_derivitivein in [ 'False', 'True']:
+    #                         args.use_derivitivein = use_derivitivein
+    #                         for dropout_prob in [0.0, 0.2, 0.8]:
+    #                             args.dropout_prob = dropout_prob
+    #
+    #                             counter = counter + 1
+    #
+    #                             if args.use_derivitive == 'True':
+    #                                 args.rep_learning_dim *= 2
+    #
+    #                             args.model_save_path = ablation_folder + '/' + str(counter) + '/'
+    #                             if not os.path.exists(args.model_save_path):
+    #                                 os.mkdir(args.model_save_path)
+    #
+    #                             conf = open(args.model_save_path + 'conf', 'w')
+    #                             str_conf = 'autoencoder_att = {}\n' \
+    #                                   'autoencoder_fixed_weight = {}\n' \
+    #                                   'autoencoder_conditioned = {}\n' \
+    #                                   'autoencoder_vae = {}\n' \
+    #                                   'autoencoder_vq = {}\n' \
+    #                                   'use_derivitivein = {}\n' \
+    #                                   'dropout_prob = {}\n'.format\
+    #                                     (
+    #                                         autoencoder_att,
+    #                                         autoencoder_fixed_weight,
+    #                                         autoencoder_conditioned,
+    #                                         autoencoder_vae,
+    #                                         autoencoder_vq,
+    #                                         use_derivitivein,
+    #                                         dropout_prob)
+    #                             conf.write(str_conf)
+    #                             conf.flush()
+    #                             conf.close()
+                                # train_epochs(args, train_loader, train_dataset,
+                                #              test_loader, None,
+                                #              pose_dim=args.rep_learning_dim,
+                                #              trial_id=trial_id)
+
+    train_epochs(args, train_loader, train_dataset, test_loader, None,
+                 pose_dim=args.rep_learning_dim, trial_id=trial_id)
+
+
+def save_config(_args):
+    if not os.path.exists(_args.model_save_path):
+        os.mkdir(_args.model_save_path)
+    conf = open(_args.model_save_path + '/conf', 'w')
+    str_conf = ''
+    for arg in vars(_args):
+        att = getattr(_args, arg)
+        if isinstance(att, list):
+            att = att[0]
+        try:
+            str_conf += str(arg) + '=' + str(att) + '\n'
+        except:
+            str_conf += str(arg) + '=' + 'None' + '\n'
+
+    conf.write(str_conf)
+    conf.flush()
+    conf.close()
+
+if __name__ == '__main__':
+    # --config =../config/seq2seq.yml
+    _args = parse_args()
+
+    if _args.use_derivitive == 'True':
+        _args.rep_learning_dim *= 2
+    save_config(_args)
+
+
+
+    main({'args': _args})
+'''
+    # Ablation study:
+    # for autoencoder_denoising in [ 'False', 'True']:
+    counter = 0
+    ablation_folder = _args.model_save_path
+    for autoencoder_att in ['False', 'True']:
+        _args.autoencoder_att = autoencoder_att
+        for autoencoder_fixed_weight in ['False', 'True']:
+            _args.autoencoder_fixed_weight = autoencoder_fixed_weight
+            for autoencoder_conditioned in ['False', 'True']:
+                _args.autoencoder_conditioned = autoencoder_conditioned
+                for autoencoder_vae in ['False', 'True']:
+                    _args.autoencoder_vae = autoencoder_vae
+                    for autoencoder_vq in ['False']:  # , 'True']:
+                        _args.autoencoder_vq = autoencoder_vq
+                        for use_derivitivein in ['False', 'True']:
+                            _args.use_derivitivein = use_derivitivein
+                            for dropout_prob in [0.0, 0.2, 0.8]:
+                                _args.dropout_prob = dropout_prob
+
+                                counter = counter + 1
+
+                                if _args.use_derivitive == 'True':
+                                    _args.rep_learning_dim *= 2
+
+                                _args.model_save_path = ablation_folder + '/' + str(counter) + '/'
+                                if not os.path.exists(_args.model_save_path):
+                                    os.mkdir(_args.model_save_path)
+
+                                conf = open(_args.model_save_path + 'conf', 'w')
+                                # str_conf = 'autoencoder_att = {}\n' \
+                                #            'autoencoder_fixed_weight = {}\n' \
+                                #            'autoencoder_conditioned = {}\n' \
+                                #            'autoencoder_vae = {}\n' \
+                                #            'autoencoder_vq = {}\n' \
+                                #            'use_derivitivein = {}\n' \
+                                #            'dropout_prob = {}\n'.format \
+                                #         (
+                                #         autoencoder_att,
+                                #         autoencoder_fixed_weight,
+                                #         autoencoder_conditioned,
+                                #         autoencoder_vae,
+                                #         autoencoder_vq,
+                                #         use_derivitivein,
+                                #         dropout_prob)
+                                str_conf = ''
+                                for arg in vars(_args):
+                                    att = getattr(_args, arg)
+                                    if isinstance(att, list):
+                                        att = att[0]
+                                    try:
+                                        str_conf += str(arg) + '=' + str(att) + '\n'
+                                    except:
+                                        str_conf += str(arg) + '=' + 'None' + '\n'
+
+                                conf.write(str_conf)
+                                conf.flush()
+                                conf.close()
+
+                                main({'args': _args})
+'''
diff --git a/scripts/train_DAE.py b/scripts/train_DAE.py
new file mode 100644
index 0000000..c99fbbb
--- /dev/null
+++ b/scripts/train_DAE.py
@@ -0,0 +1,475 @@
+import datetime
+import os
+import pprint
+import random
+import time
+import sys
+import numpy as np
+import torch
+from sklearn.cluster import KMeans
+
+from torch.utils.data import DataLoader
+
+[sys.path.append(i) for i in ['.', '..']]
+
+from model import vocab
+from model.seq2seq_net import Seq2SeqNet
+from train_eval.train_seq2seq import train_iter_seq2seq, train_iter_DAE
+from utils.average_meter import AverageMeter
+from utils.vocab_utils import build_vocab
+
+from config.parse_args import parse_args
+
+from torch import optim
+
+from data_loader.lmdb_data_loader import *
+import utils.train_utils
+
+import seaborn as sns
+from sklearn.decomposition import PCA
+from openTSNE import TSNE
+import matplotlib.pyplot as plt
+from scipy.signal import savgol_filter
+
+from model.DAE_model import DAE_Network, VQ_Frame, VAE_Network
+import pickle
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+global Global_loss_train
+global Global_loss_eval
+
+Train_Deoising = False
+
+def init_model(args, lang_model, pose_dim, _device):
+    n_frames = args.n_poses
+    # generator = Seq2SeqNet(args, pose_dim, n_frames, lang_model.n_words, args.wordembed_dim,
+    #                        lang_model.word_embedding_weights).to(_device)
+    motion_dim = 162#162 #135
+    # try:
+    if args.autoencoder_vq=='True':
+        Network = VQ_Frame(motion_dim, args.hidden_size,
+                           args.autoencoder_vae=='True',
+                           int(args.autoencoder_vq_components))
+    elif args.autoencoder_vae == 'True':
+        Network = VAE_Network(motion_dim, args.hidden_size)
+
+    else:
+        Network = DAE_Network(motion_dim, args.hidden_size)
+
+    # except:
+    #     Network = DAE_Network(motion_dim, args.hidden_size)
+
+    loss_fn = torch.nn.MSELoss()
+    Network = Network.to(device)
+    return Network, loss_fn
+
+
+def train_epochs(args, train_data_loader, test_data_loader, lang_model, pose_dim, trial_id=None):
+    global Global_loss_train
+    global Global_loss_eval
+    start = time.time()
+    loss_meters = [AverageMeter('loss'), AverageMeter('var_loss')]
+
+    # interval params
+    print_interval = int(len(train_data_loader) )
+    save_model_epoch_interval = args.epochs
+
+    # init model
+    generator, loss_fn = init_model(args, lang_model, pose_dim, device)
+
+    if args.hidden_size == -1:
+        try:  # multi gpu
+            gen_state_dict = generator.module.state_dict()
+        except AttributeError:  # single gpu
+            gen_state_dict = generator.state_dict()
+
+        save_name = '{}/{}_H{}_checkpoint_{:03d}.bin'.format(args.model_save_path, args.name, args.hidden_size, 20)
+
+        utils.train_utils.save_checkpoint({
+            'args': args, 'epoch': 20, 'lang_model': lang_model,
+            'pose_dim': pose_dim, 'gen_dict': gen_state_dict
+        }, save_name)
+        return
+
+
+
+    # define optimizers
+    gen_optimizer = optim.Adam(generator.parameters(), lr=args.learning_rate, betas=(0.5, 0.999))
+
+    # training
+    global_iter = 0
+    train_res_recon_error = []
+    train_res_perplexity = []
+    print(args.epochs)
+    for epoch in range(1, args.epochs+1):
+        if(args.autoencoder_vq=='True'):
+            plot_embedding(args, generator, epoch,
+                           train_res_recon_error,
+                           train_res_perplexity)
+
+        # evaluate the test set
+        # val_metrics = evaluate_testset(test_data_loader, generator, loss_fn, args)
+        val_metrics = 0.01
+        Global_loss_eval.append(val_metrics)
+        # save model
+        if epoch % save_model_epoch_interval == 0 and epoch > 0:
+            try:  # multi gpu
+                gen_state_dict = generator.module.state_dict()
+            except AttributeError:  # single gpu
+                gen_state_dict = generator.state_dict()
+
+            save_name = '{}/{}_H{}_checkpoint_{:03d}.bin'.format(args.model_save_path, args.name, args.hidden_size, epoch)
+
+            utils.train_utils.save_checkpoint({
+                'args': args, 'epoch': epoch, 'lang_model': lang_model,
+                'pose_dim': pose_dim, 'gen_dict': gen_state_dict
+            }, save_name)
+
+        # VQ Tricks
+        Use_Tricks = False
+        if args.autoencoder_vq == 'True' and (not Use_Tricks):
+            generator.skip_vq = False
+        if args.autoencoder_vq == 'True' and Use_Tricks:
+            StartVQ_Epoch = 5
+            ReEstimate_Epoch = 5
+            if epoch < StartVQ_Epoch:
+                generator.skip_vq = True
+            else:
+                generator.skip_vq = False
+            if epoch % ReEstimate_Epoch == 0 and (not generator.skip_vq):
+                generator.train(False)
+                all_latents = []
+                for iter_idx, data in enumerate(train_data_loader, 0):
+                    noisy, original = data
+                    original = original.to(device)
+                    # print(original.shape)
+                    original = torch.squeeze(original)
+                    current_batch_latents = generator.encoder(original)
+                    all_latents.append(current_batch_latents)
+                all_latents = torch.vstack(all_latents)
+                all_latents = all_latents.cpu().detach().numpy()
+                generator.train(True)
+                normalized_data = all_latents
+
+                km = KMeans(n_clusters=generator.vq_components, max_iter=2500,
+                            random_state=0).fit(normalized_data)
+                print("Kmeans trained!")
+                # print("km",km.cluster_centers_.shape)
+                # print("Embeddings", generator.vq_layer._embedding.weight.shape)
+                generator.vq_layer._embedding.weight.data = ( torch.tensor(km.cluster_centers_).to(device) )
+                # exit()
+
+        # train iter
+        iter_start_time = time.time()
+        for iter_idx, data in enumerate(train_data_loader, 0):
+            global_iter += 1
+            # in_text, text_lengths, target_vec, in_audio, aux_info = data
+            noisy, original = data
+            batch_size = original.size(0)
+
+            noisy = noisy.to(device)
+            original = original.to(device)
+
+            # train
+            if args.autoencoder_vq=='True':
+                loss, perplexity = train_iter_DAE(args, epoch, original, original, generator, gen_optimizer)
+                train_res_recon_error.append(loss['loss'])
+                train_res_perplexity.append(perplexity.item())
+            if Train_Deoising:
+                loss = train_iter_DAE(args, epoch, noisy, original, generator, gen_optimizer)
+            else:
+                loss = train_iter_DAE(args, epoch, original, original, generator, gen_optimizer)
+
+
+            # loss values
+            for loss_meter in loss_meters:
+                name = loss_meter.name
+                if name in loss:
+                    loss_meter.update(loss[name], batch_size)
+
+            # print training status
+            if (iter_idx + 1) % print_interval == 0:
+                print_summary = 'EP {} ({:3d}) | {:>8s}, {:.0f} samples/s | '.format(
+                    epoch, iter_idx + 1, utils.train_utils.time_since(start),
+                    batch_size / (time.time() - iter_start_time))
+
+                for loss_meter in loss_meters:
+                    if loss_meter.count > 0:
+                        print_summary += '{}: {:.3f}, '.format(loss_meter.name, loss_meter.avg)
+                        Global_loss_train[loss_meter.name].append(loss_meter.avg)
+                        loss_meter.reset()
+                logging.info(print_summary)
+
+
+            iter_start_time = time.time()
+    print("___________")
+    # print(Global_loss_train)
+
+def evaluate_testset(test_data_loader, generator, loss_fn, args):
+    # to evaluation mode
+    generator.train(False)
+    global Global_loss_eval
+    losses = AverageMeter('loss')
+    start = time.time()
+
+    with torch.no_grad():
+        for iter_idx, data in enumerate(test_data_loader, 0):
+            # in_text, text_lengths, target_vec, in_audio, aux_info = data
+            noisy, original = data
+            batch_size = original.size(0)
+
+            noisy = noisy.to(device)
+            original = original.to(device)
+
+            if args.autoencoder_vq == 'True' and args.autoencoder_vae == 'True':
+                out_poses, vq_loss, preplexity, logvarm, meu = generator(noisy)
+            elif args.autoencoder_vq == 'True' and args.autoencoder_vae == 'False':
+                out_poses, vq_loss, preplexity = generator(noisy)
+            elif args.autoencoder_vq == 'False' and args.autoencoder_vae == 'True':
+                out_poses, logvarm, meu = generator(noisy)
+            else:
+                out_poses = generator(noisy)
+
+            # This one is for GSOFT VQ
+            # loss = out_poses.log_prob(original).sum(dim=1).mean()
+            loss = loss_fn(out_poses, original)
+
+            losses.update(loss.item(), batch_size)
+
+    # back to training mode
+    generator.train(True)
+
+    # print
+    elapsed_time = time.time() - start
+    logging.info('[VAL] loss: {:.3f} / {:.1f}s'.format(losses.avg, elapsed_time))
+
+    return losses.avg
+
+
+def main(config, train_loader, test_loader, lang_model, H, vq_ncomponent=1):
+    args = config['args']
+    args.hidden_size = H
+    args.autoencoder_vq_components = vq_ncomponent
+    if args.autoencoder_vq=='True': # ths was for training a lot of networks
+        args.model_save_path +='/'+str(vq_ncomponent)
+    # args.epochs = 20
+    args.model_save_path += "/train_DAE_H" + str(H)
+    trial_id = None
+
+    # random seed
+    if args.random_seed >= 0:
+        torch.manual_seed(args.random_seed)
+        torch.cuda.manual_seed_all(args.random_seed)
+        np.random.seed(args.random_seed)
+        random.seed(args.random_seed)
+        os.environ['PYTHONHASHSEED'] = str(args.random_seed)
+
+    # set logger
+    utils.train_utils.set_logger(args.model_save_path, os.path.basename(__file__).replace('.py', '.log'))
+
+    logging.info("PyTorch version: {}".format(torch.__version__))
+    logging.info("CUDA version: {}".format(torch.version.cuda))
+    logging.info(pprint.pformat(vars(args)))
+
+    # Dataset moved from here.
+
+    global Global_loss_train
+    Global_loss_train= {"loss": [], "var_loss":[]}
+    global Global_loss_eval
+    Global_loss_eval = []
+    # train
+    train_epochs(args, train_loader, test_loader, lang_model,
+                 pose_dim=15*9, trial_id=trial_id)
+
+    # print("****************")
+    # print(Global_loss_eval)
+    dict2save = {"Global_train": Global_loss_train, "Global_eval": Global_loss_eval}
+    pickle.dump(dict2save, open(args.model_save_path+'/globals.pk', 'wb'))
+
+    try:
+        plot_loss(Global_loss_train, Global_loss_eval, args.model_save_path+'/loss_plot.png')
+    except:
+        pass
+    # plt.plot(Global_loss_train)
+
+    return Global_loss_train['loss'][-1], Global_loss_eval[-1]
+
+def plot_loss(Global_loss_train, Global_loss_eval, save_path):
+    plt.plot(Global_loss_train["loss"], label='Train loss')
+    plt.plot(Global_loss_eval, label='Evaluation loss')
+
+    # Labeling the X-axis
+    plt.xlabel('Epoch number')
+    # Labeling the Y-axis
+    plt.ylabel('Loss Average')
+    # Give a title to the graph
+    plt.title('Training/Evaluation Loss based on epoch number \n' +
+              "{:.4f}".format(Global_loss_train["loss"][-1]))
+
+    # Show a legend on the plot
+    plt.legend()
+
+
+    plt.savefig(save_path)
+    # plt.savefig(os.path.join(args.model_save_path, 'loss_plot.png'))
+    plt.show()
+
+def plot_embedding(args, rnn, epoch_num,
+                   train_res_recon_error,
+                   train_res_perplexity):
+
+
+    w = rnn.vq_layer._embedding.weight.cpu().detach().numpy()
+    # PCA
+    if w.shape[0]>20:
+        pca = PCA(n_components=20)
+        priciple_components = pca.fit(w)
+        normalized_data = priciple_components.transform(w)
+    else:
+        normalized_data = w
+    # TSNE
+
+    MyTSNE = TSNE(
+        n_components=2,
+        perplexity=30,
+        metric='euclidean',
+        n_jobs=8,
+        verbose=True
+    )
+    X_embedded = MyTSNE.fit(normalized_data)
+
+    plt.figure(figsize=(16, 10))
+    # palette = sns.color_palette("bright", k_component)
+    # 2D
+
+    sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1],
+                    # hue=labels_,
+                    legend=False)
+    plt.title("Embedding visualization" + str(epoch_num))
+    address = (args.model_save_path) + '/plots/'
+    os.makedirs(address, exist_ok=True)
+    address += 'Embedding_Epoch(' + str(epoch_num) +').png'
+    plt.savefig(address)
+    plt.clf()
+    plt.show()
+
+
+    # Reconstruction loss and perplexity
+    print(train_res_perplexity)
+    try:
+        train_res_recon_error_smooth = savgol_filter(train_res_recon_error, 201, 7)
+        train_res_perplexity_smooth =  savgol_filter(train_res_perplexity, 201, 7)
+    except:
+        train_res_recon_error_smooth = train_res_recon_error #savgol_filter(train_res_recon_error, 201, 7)
+        train_res_perplexity_smooth = train_res_perplexity #[0] #savgol_filter(train_res_perplexity, 201, 7)
+
+    f = plt.figure(figsize=(16, 8))
+    ax = f.add_subplot(1, 2, 1)
+    ax.plot(train_res_recon_error_smooth)
+    ax.set_yscale('log')
+    ax.set_title('Smoothed NMSE.')
+    ax.set_xlabel('iteration')
+
+    ax = f.add_subplot(1, 2, 2)
+    ax.plot(train_res_perplexity_smooth)
+    ax.set_title('Smoothed Average codebook usage (perplexity).')
+    ax.set_xlabel('iteration')
+
+    address = (args.model_save_path) + '/plots/Error_Epoch(' + str(epoch_num) + ').png'
+
+    plt.savefig(address)
+    plt.clf()
+    plt.show()
+
+
+
+
+    pass
+
+
+
+if __name__ == '__main__':
+
+    # --config=../config/seq2seq.yml
+    _args = parse_args()
+    # dataset
+    args = _args
+    train_dataset = TrinityDataset_DAE(args, args.train_data_path[0],
+                                       n_poses=args.n_poses,
+                                       subdivision_stride=args.subdivision_stride,
+                                       pose_resampling_fps=args.motion_resampling_framerate,
+                                       data_mean=args.data_mean, data_std=args.data_std)
+    train_loader = DataLoader(dataset=train_dataset, batch_size=args.batch_size,
+                              shuffle=True, drop_last=True, num_workers=args.loader_workers, pin_memory=True
+                              # collate_fn=word_seq_collate_fn
+                              )
+
+    val_dataset = TrinityDataset_DAE(args, args.val_data_path[0],
+                                     n_poses=args.n_poses,
+                                     subdivision_stride=args.subdivision_stride,
+                                     pose_resampling_fps=args.motion_resampling_framerate,
+                                     data_mean=args.data_mean, data_std=args.data_std)
+    test_loader = DataLoader(dataset=val_dataset, batch_size=args.batch_size,
+                             shuffle=False, drop_last=True, num_workers=args.loader_workers, pin_memory=True
+                             # collate_fn=word_seq_collate_fn
+                             )
+
+    # build vocab
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    lang_model = build_vocab('words', [train_dataset, val_dataset], vocab_cache_path, args.wordembed_path,
+                             args.wordembed_dim)
+    train_dataset.set_lang_model(lang_model)
+    val_dataset.set_lang_model(lang_model)
+
+    train_loss_list = []
+    eval_loss_list = []
+    dim_lost = []
+    backupsv = _args.model_save_path
+    for k in range(45, 46, 1):
+        print("*******************************************************************")
+        print("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^")
+        print("k=" + str(k))
+        _args.model_save_path = backupsv
+        current_train_loss, current_eval_loss = main({'args': _args}, train_loader, test_loader, lang_model, k,
+             vq_ncomponent=k)
+        dim_lost.append(int(k))
+        train_loss_list.append(current_train_loss)
+        eval_loss_list.append(current_eval_loss)
+
+        # Do inference
+        save_name = '{}/{}_H{}_checkpoint_{:03d}.bin'.format(_args.model_save_path, _args.name,
+                                                             _args.hidden_size, _args.epochs)
+        command = 'python inference_DAE.py ' + save_name
+        os.system(command)
+
+
+
+    #     Plot results
+    plt.plot(dim_lost, train_loss_list, label='Train loss')
+    plt.plot(dim_lost, eval_loss_list, label='Evaluation loss')
+
+    # Labeling the X-axis
+    plt.xlabel('Dimensionality')
+    # Labeling the Y-axis
+    plt.ylabel('Loss Average')
+    # Give a title to the graph
+    plt.title('Training/Evaluation Loss based on latent dim')
+
+
+
+    # Show a legend on the plot
+    plt.legend()
+
+    plt.savefig(backupsv + '/overall.png')
+    # plt.savefig(os.path.join(args.model_save_path, 'loss_plot.png'))
+    plt.show()
+
+
+
+
+
+
+    '''
+    main({'args': _args}, train_loader, test_loader, lang_model, 150, #-2 #-1,
+             vq_ncomponent=args.autoencoder_vq_components)
+    '''
\ No newline at end of file
diff --git a/scripts/train_autoencoder_VQVAE.py b/scripts/train_autoencoder_VQVAE.py
new file mode 100644
index 0000000..bb9ad04
--- /dev/null
+++ b/scripts/train_autoencoder_VQVAE.py
@@ -0,0 +1,505 @@
+import datetime
+import os
+import pprint
+import random
+import time
+import sys
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.signal import savgol_filter
+
+from torch.utils.data import DataLoader
+from tqdm import tqdm
+import torch
+[sys.path.append(i) for i in ['.', '..']]
+
+from model import vocab
+from model.seq2seq_net import Seq2SeqNet
+from model.Autoencoder_VQVAE_model import Autoencoder_VQVAE
+from train_eval.train_seq2seq import train_iter_seq2seq, train_iter_Autoencoder_seq2seq, train_iter_Autoencoder_ssl_seq2seq, train_iter_Autoencoder_VQ_seq2seq
+from utils.average_meter import AverageMeter
+from utils.vocab_utils import build_vocab
+
+from config.parse_args import parse_args
+
+from torch import optim
+
+from data_loader.lmdb_data_loader import *
+import utils.train_utils
+
+import utils
+from pymo.preprocessing import *
+from pymo.viz_tools import *
+from pymo.writers import *
+from utils.data_utils import SubtitleWrapper, normalize_string
+import joblib as jl
+
+import seaborn as sns
+from sklearn.decomposition import PCA
+from openTSNE import TSNE
+import contextlib
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+debug = False
+
+def init_model(args, lang_model, pose_dim, _device):
+    n_frames = args.n_poses
+    generator = Autoencoder_VQVAE(args, pose_dim, n_frames).to(_device)
+    loss_fn = torch.nn.MSELoss()
+
+    return generator, loss_fn
+
+
+
+
+def train_epochs(args, train_data_loader, train_sim_dataset, test_data_loader, lang_model, pose_dim, trial_id=None):
+    start = time.time()
+    loss_meters = [AverageMeter('loss'), AverageMeter('var_loss')]
+
+    if args.use_derivitive=='True':
+        pass
+
+    # interval params
+    print_interval = int(len(train_data_loader))
+    save_model_epoch_interval = args.epochs
+
+    # init model
+    generator, loss_fn = init_model(args, lang_model, pose_dim, device)
+    start_epoch = 1
+    load_pretrained = False
+    if load_pretrained:
+        checkpoint_path_rnn = '{}/{}_checkpoint_{:03d}.bin'.format(args.model_save_path, args.name, 20)
+        args, generator, loss_fn, lang_model, out_dim = utils.train_utils.load_checkpoint_and_model(
+            checkpoint_path_rnn, device, 'autoencoder_vq')
+        start_epoch = 20
+
+
+    # Here, we freeze the encoder to train the decoder
+    if args.autoencoder_freeze_encoder == 'True':
+        rnn = utils.train_utils.load_checkpoint_and_model(
+            args.autoencoder_checkpoint, device)
+        args_rnn, rnn, loss_fn, lang_model, out_dim = rnn
+        generator = rnn
+        generator.freez_encoder()
+        generator.autoencoder_fixed_weight = False
+        for param in generator.decoder.parameters():
+            param.requires_grad = True
+
+    # define optimizers
+    gen_optimizer = optim.Adam(generator.parameters(), lr=args.learning_rate, betas=(0.5, 0.999))
+
+    val_metrics = evaluate_testset(test_data_loader, generator, loss_fn, args)
+
+
+    # To record loss and evaluations metric:
+    val_metrics_list = []
+    loss_list = []
+
+
+    # training
+    global_iter = 0
+
+    train_res_recon_error = []
+    train_res_perplexity = []
+    # with open(os.devnull, "w") as f, contextlib.redirect_stdout(f):
+    #     plot_embedding(args, generator, 0,
+    #                    train_res_recon_error,
+    #                    train_res_perplexity)
+    for epoch in range(start_epoch, args.epochs+1):
+
+        if args.autoencoder_vq == 'True':
+            with open(os.devnull, "w") as f, contextlib.redirect_stdout(f):
+                plot_embedding(args, generator, epoch,
+                               train_res_recon_error,
+                               train_res_perplexity)
+
+        # evaluate the test set
+        val_metrics = evaluate_testset(test_data_loader, generator, loss_fn, args)
+        val_metrics_list.append(val_metrics)
+        # save model
+        if epoch % save_model_epoch_interval == 0 and epoch > 0:
+            try:  # multi gpu
+                gen_state_dict = generator.module.state_dict()
+            except AttributeError:  # single gpu
+                gen_state_dict = generator.state_dict()
+
+            save_name = '{}/{}_checkpoint_{:03d}.bin'.format(args.model_save_path, args.name, epoch)
+
+            utils.train_utils.save_checkpoint({
+                'args': args,
+                'epoch': epoch,
+                'lang_model': lang_model,
+                'pose_dim': pose_dim,
+                'gen_dict': gen_state_dict,
+                'val_metrics_list': val_metrics_list,
+                'loss_list': loss_list
+
+            }, save_name)
+
+        # train iter
+        iter_start_time = time.time()
+        loss_epoch = AverageMeter('loss')
+        loss_epoch.reset()
+        # Todo: move to args
+        train_labeled = False
+        labeled_count = 20
+        for iter_idx, data in enumerate(train_data_loader, 0):
+            global_iter += 1
+            encoded_input, encoded_output = data
+            batch_size = encoded_output.size(0)
+
+            encoded_input = encoded_input.to(device)
+            encoded_output = encoded_output.to(device)
+
+            # train
+
+            if train_labeled:
+                stack_pairs1, stack_pairs2, stack_labels = train_sim_dataset.get_labeled_(labeled_count)
+
+                if debug:
+                    print("STAAAAACK", stack_pairs1.shape)
+                    print("STAAAAACK_LLL", stack_labels.shape)
+                stack_pairs1 = stack_pairs1.to(device)
+                stack_pairs2 = stack_pairs2.to(device)
+                stack_labels = stack_labels.to(device)
+
+
+                loss = train_iter_Autoencoder_ssl_seq2seq(args, epoch, encoded_input, encoded_output, generator,
+                                                      gen_optimizer,
+                                                          stack_pairs1, stack_pairs2, stack_labels)
+            else:
+                if args.autoencoder_vq == 'True':
+                    loss, perplexity = train_iter_Autoencoder_VQ_seq2seq(args, epoch,
+                                                          encoded_input, encoded_output, generator, gen_optimizer)
+                    train_res_recon_error.append(loss['loss'])
+                    train_res_perplexity.append(perplexity.item())
+                else:
+                    loss = train_iter_Autoencoder_VQ_seq2seq(args, epoch,
+                                                                         encoded_input, encoded_output, generator,
+                                                                         gen_optimizer)
+
+            # Visualization for debug
+
+
+
+            loss_epoch.update(loss['loss'], batch_size)
+            # loss values
+            for loss_meter in loss_meters:
+                name = loss_meter.name
+                if name in loss:
+                    loss_meter.update(loss[name], batch_size)
+
+            # print training status
+            if (iter_idx + 1) % print_interval == 0:
+                print_summary = 'EP {} ({:3d}) | {:>8s}, {:.0f} samples/s | '.format(
+                    epoch, iter_idx + 1, utils.train_utils.time_since(start),
+                    batch_size / (time.time() - iter_start_time))
+                for loss_meter in loss_meters:
+                    if loss_meter.count > 0:
+                        print_summary += '{}: {:.3f}, '.format(loss_meter.name, loss_meter.avg)
+                        # if args.autoencoder_vq == 'True':
+                        #     print_summary += '\nperplexity,' + str(perplexity.detach().cpu().numpy())
+                        loss_meter.reset()
+                logging.info(print_summary)
+
+            iter_start_time = time.time()
+
+        loss_list.append(loss_epoch.avg)
+
+        # if args.autoencoder_vq == 'True':
+        #     with open(os.devnull, "w") as f, contextlib.redirect_stdout(f):
+        #         plot_embedding(args, generator, epoch,
+        #                        train_res_recon_error,
+        #                        train_res_perplexity)
+
+    plot_loss(args, all_eval_loss=val_metrics_list, all_train_loss=loss_list)
+
+
+def evaluate_testset(test_data_loader, generator, loss_fn, args):
+    # to evaluation mode
+    generator.train(False)
+
+    losses = AverageMeter('loss')
+    start = time.time()
+
+    with torch.no_grad():
+        for iter_idx, data in enumerate(test_data_loader, 0):
+            in_poses, target_poses = data
+            batch_size = in_poses.size(0)
+            if debug:
+                print("in_shape", in_poses.shape, "Target_poses", target_poses.shape)
+            in_poses = in_poses.to(device)
+            target_poses = target_poses.to(device)
+            if args.autoencoder_vq == 'True':
+                if args.autoencoder_vae== 'True':
+                    out_poses, latent_, mue, logvar, loss_vq, perplexity_vq= generator(in_poses, target_poses)
+                else:
+                    out_poses, latent_, loss_vq, perplexity_vq = generator(in_poses, target_poses)
+            else:
+                if args.autoencoder_vae== 'True':
+                    out_poses, latent_, mue, logvar = generator(in_poses, target_poses)
+                else:
+                    out_poses, latent_ = generator(in_poses, target_poses)
+            # print("out_poses", out_poses.shape, "Target_poses", target_poses.shape)
+            loss = loss_fn(out_poses, target_poses)
+            losses.update(loss.item(), batch_size)
+
+    # back to training mode
+    generator.train(True)
+
+    # print
+    elapsed_time = time.time() - start
+    logging.info('[VAL] loss: {:.3f} / {:.1f}s'.format(losses.avg, elapsed_time))
+
+    return losses.avg
+
+
+
+def plot_loss(args, all_eval_loss, all_train_loss):
+
+
+    # X = np.arange(136-3)
+    # fig = plt.figure()
+    # ax = fig.add_axes([0, 0, 1, 1])
+    # ax.bar(X + 0.00, all_train_loss[0], color='b', width=0.25)
+    # ax.bar(X + 0.25, all_eval_loss[1], color='g', width=0.25)
+    # plt.show()
+    # plotting the second plot
+    plt.plot(all_train_loss, label='Train loss')
+    plt.plot(all_eval_loss, label='Evaluation loss')
+
+    # Labeling the X-axis
+    plt.xlabel('Epoch number')
+    # Labeling the Y-axis
+    plt.ylabel('Loss Average')
+    # Give a title to the graph
+    plt.title('Training/Evaluation Loss based on epoch number')
+
+    # Show a legend on the plot
+    plt.legend()
+
+    plt.savefig(os.path.join(args.model_save_path, 'loss_plot.png'))
+    plt.show()
+
+def plot_embedding(args, rnn, epoch_num,
+                   train_res_recon_error,
+                   train_res_perplexity):
+    os.makedirs(args.model_save_path + '/plots', exist_ok=True)
+
+    w = rnn.vq_layer._embedding.weight.cpu().detach().numpy()
+    # PCA
+    try:
+        pca = PCA(n_components=50)
+        priciple_components = pca.fit(w)
+        normalized_data = priciple_components.transform(w)
+    except:
+        normalized_data = w
+    # TSNE
+
+    MyTSNE = TSNE(
+        n_components=2,
+        perplexity=30,
+        metric='euclidean',
+        n_jobs=8,
+        verbose=True
+    )
+    X_embedded = MyTSNE.fit(normalized_data)
+
+    plt.figure(figsize=(16, 10))
+    # palette = sns.color_palette("bright", k_component)
+    # 2D
+
+    sns.scatterplot(X_embedded[:, 0], X_embedded[:, 1],
+                    # hue=labels_,
+                    legend=False)
+    plt.title("Embedding visualization" + str(epoch_num))
+    
+    address = (args.model_save_path) + '/plots/Embedding_Epoch(' + str(epoch_num) +').png'
+    plt.savefig(address)
+    plt.clf()
+    plt.show()
+
+
+    # Reconstruction loss and perplexity
+    print(train_res_perplexity)
+    try:
+        train_res_recon_error_smooth = savgol_filter(train_res_recon_error, 201, 7)
+        train_res_perplexity_smooth = savgol_filter(train_res_perplexity, 201, 7)
+    except:
+        train_res_recon_error_smooth = train_res_recon_error  # savgol_filter(train_res_recon_error, 201, 7)
+        train_res_perplexity_smooth = train_res_perplexity  # [0] #savgol_filter(train_res_perplexity, 201, 7)
+
+    f = plt.figure(figsize=(16, 8))
+    ax = f.add_subplot(1, 2, 1)
+    ax.plot(train_res_recon_error_smooth)
+    ax.set_yscale('log')
+    val = "?"
+    if len(train_res_recon_error_smooth)>0:
+        val = str(train_res_recon_error_smooth[-1])
+    ax.set_title('Smoothed NMSE: ' + val)
+    ax.set_xlabel('iteration')
+
+    ax = f.add_subplot(1, 2, 2)
+    ax.plot(train_res_perplexity_smooth)
+
+    val = "?"
+    if len(train_res_perplexity_smooth)>0:
+        val += str(train_res_perplexity_smooth[-1])
+    ax.set_title('Smoothed Average codebook usage (perplexity): ' + val)
+    ax.set_xlabel('iteration')
+
+    address = (args.model_save_path) + '/plots/Error_Epoch(' + str(epoch_num) + ').png'
+
+    plt.savefig(address)
+    plt.clf()
+    plt.show()
+
+
+
+
+    pass
+
+def save_sim_graph_gesture(args, pairs, data_original):
+    save_result_path = args.model_save_path
+    sim_count = 0
+    dissim_count = 0
+    for item in tqdm(pairs):
+        if item[2] == 0:
+            dissim_count += 1
+            directory = str(save_result_path) + '/sim_graph/dis/' + str(dissim_count)
+            os.makedirs(directory)
+            make_bvh(directory, str(item[0]), data_original[item[0]])
+            make_bvh(directory, str(item[1]), data_original[item[1]])
+        else:
+            sim_count = sim_count + 1
+            directory = str(save_result_path) + '/sim_graph/sim/' + str(sim_count)
+            os.makedirs(directory)
+            make_bvh(directory, str(item[0]), data_original[item[0]])
+            make_bvh(directory, str(item[1]), data_original[item[1]])
+def make_bvh(save_path, filename_prefix, poses):
+    writer = BVHWriter()
+    pipeline = jl.load('../resource/data_pipe.sav')
+
+    # smoothing
+    n_poses = poses.shape[0]
+    out_poses = np.zeros((n_poses, poses.shape[1]))
+
+    for i in range(poses.shape[1]):
+        out_poses[:, i] = savgol_filter(poses[:, i], 15, 2)  # NOTE: smoothing on rotation matrices is not optimal
+
+    # rotation matrix to euler angles
+    out_poses = out_poses.reshape((out_poses.shape[0], -1, 9))
+    out_poses = out_poses.reshape((out_poses.shape[0], out_poses.shape[1], 3, 3))
+    out_euler = np.zeros((out_poses.shape[0], out_poses.shape[1] * 3))
+    for i in range(out_poses.shape[0]):  # frames
+        r = R.from_matrix(out_poses[i])
+        out_euler[i] = r.as_euler('ZXY', degrees=True).flatten()
+
+    bvh_data = pipeline.inverse_transform([out_euler])
+
+    out_bvh_path = os.path.join(save_path, filename_prefix + '_generated.bvh')
+    with open(out_bvh_path, 'w') as f:
+        writer.write(bvh_data[0], f)
+
+
+
+
+
+def main(config):
+    args = config['args']
+
+    trial_id = None
+
+    # random seed
+    if args.random_seed >= 0:
+        torch.manual_seed(args.random_seed)
+        torch.cuda.manual_seed_all(args.random_seed)
+        np.random.seed(args.random_seed)
+        random.seed(args.random_seed)
+        os.environ['PYTHONHASHSEED'] = str(args.random_seed)
+
+    # set logger
+    utils.train_utils.set_logger(args.model_save_path, os.path.basename(__file__).replace('.py', '.log'))
+
+    logging.info("PyTorch version: {}".format(torch.__version__))
+    logging.info("CUDA version: {}".format(torch.version.cuda))
+    logging.info(pprint.pformat(vars(args)))
+
+    # dataset
+    # Todo: change it backt to the real train
+    train_dataset = TrinityDataset_DAEed_Autoencoder(args, args.train_data_path[0],
+                                   n_poses=args.n_poses,
+                                   subdivision_stride=args.subdivision_stride,
+                                   pose_resampling_fps=args.motion_resampling_framerate,
+                                   data_mean=args.data_mean, data_std=args.data_std)
+
+    if args.use_similarity == 'True':
+        print()
+        train_dataset.creat_similarity_dataset(args.data_for_sim, args.similarity_labels)
+    #     Sanity check
+    #     save_sim_graph_gesture(args, train_dataset.pairwise_labels, train_dataset.data_original)
+    #     print("Sanity check saved!")
+
+
+    train_loader = DataLoader(dataset=train_dataset, batch_size=args.batch_size,
+                              shuffle=True, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                              # collate_fn=word_seq_collate_fn
+                              )
+
+
+    val_dataset = TrinityDataset_DAEed_Autoencoder(args, args.val_data_path[0],
+                                 n_poses=args.n_poses,
+                                 subdivision_stride=args.subdivision_stride,
+                                 pose_resampling_fps=args.motion_resampling_framerate,
+                                 data_mean=args.data_mean, data_std=args.data_std)
+
+    test_loader = DataLoader(dataset=val_dataset, batch_size=args.batch_size,
+                             shuffle=False, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                             # collate_fn=word_seq_collate_fn
+                             )
+
+    # build vocab
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    # lang_model = build_vocab('words', [train_dataset, val_dataset], vocab_cache_path, args.wordembed_path,
+    #                          args.wordembed_dim)
+    # train_dataset.set_lang_model(lang_model)
+    # val_dataset.set_lang_model(lang_model)
+
+    # train
+
+    train_epochs(args, train_loader, train_dataset, test_loader, None,
+                 pose_dim=args.rep_learning_dim, trial_id=trial_id)
+
+    # Do inference
+
+    checkpoint_path_rnn = '{}/{}_checkpoint_{:03d}.bin'.format(args.model_save_path, args.name, args.epochs)
+    checkpoint_path_DAE = args.rep_learning_checkpoint
+    command = 'python inference_Autoencoder.py ' + checkpoint_path_DAE + ' ' + checkpoint_path_rnn
+    os.system(command)
+
+def save_config(_args):
+    if not os.path.exists(_args.model_save_path):
+        os.mkdir(_args.model_save_path)
+    conf = open(_args.model_save_path + '/conf', 'w')
+    str_conf = ''
+    for arg in vars(_args):
+        att = getattr(_args, arg)
+        if isinstance(att, list):
+            att = att[0]
+        try:
+            str_conf += str(arg) + '=' + str(att) + '\n'
+        except:
+            str_conf += str(arg) + '=' + 'None' + '\n'
+
+    conf.write(str_conf)
+    conf.flush()
+    conf.close()
+
+if __name__ == '__main__':
+    # --config =../config/seq2seq.yml
+    _args = parse_args()
+
+    if _args.use_derivitive == 'True':
+        _args.rep_learning_dim *= 2
+    save_config(_args)
+    main({'args': _args})
diff --git a/scripts/train_cluster2gesture.py b/scripts/train_cluster2gesture.py
new file mode 100644
index 0000000..69f3b25
--- /dev/null
+++ b/scripts/train_cluster2gesture.py
@@ -0,0 +1,192 @@
+import datetime
+import pprint
+import random
+import time
+import sys
+import numpy as np
+
+from torch.utils.data import DataLoader
+import torch
+[sys.path.append(i) for i in ['.', '..']]
+
+from model import vocab
+from model.seq2seq_net import Seq2SeqNet
+from model.seq2seq_with_cluster_model import cluster2gesture_model
+from train_eval.train_seq2seq import train_iter_seq2seq, train_iter_c2g_seq2seq
+from utils.average_meter import AverageMeter
+from utils.vocab_utils import build_vocab
+
+from config.parse_args import parse_args
+
+from torch import optim
+
+from data_loader.lmdb_data_loader import *
+import utils.train_utils
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+
+def init_model(args, lang_model, pose_dim, _device):
+    n_frames = args.n_poses
+    generator = cluster2gesture_model(args, 40, args.hidden_size, args.hidden_size, args.rep_learning_dim, ).to(_device)
+    loss_fn = torch.nn.MSELoss()
+
+    return generator, loss_fn
+
+
+def train_epochs(args, train_data_loader, test_data_loader, lang_model, pose_dim, trial_id=None):
+    start = time.time()
+    loss_meters = [AverageMeter('loss'), AverageMeter('var_loss')]
+
+    # interval params
+    print_interval = int(len(train_data_loader) / 5)
+    save_model_epoch_interval = 20
+
+    # init model
+    generator, loss_fn = init_model(args, lang_model, pose_dim, device)
+
+    # define optimizers
+    gen_optimizer = optim.Adam(generator.parameters(), lr=args.learning_rate, betas=(0.5, 0.999))
+
+    # training
+    global_iter = 0
+    for epoch in range(1, args.epochs+1):
+        # evaluate the test set
+        val_metrics = evaluate_testset(test_data_loader, generator, loss_fn, args)
+
+        # save model
+        if epoch % save_model_epoch_interval == 0 and epoch > 0:
+            try:  # multi gpu
+                gen_state_dict = generator.module.state_dict()
+            except AttributeError:  # single gpu
+                gen_state_dict = generator.state_dict()
+
+            save_name = '{}/{}_checkpoint_{:03d}.bin'.format(args.model_save_path, args.name, epoch)
+
+            utils.train_utils.save_checkpoint({
+                'args': args, 'epoch': epoch, 'lang_model': lang_model,
+                'pose_dim': pose_dim, 'gen_dict': gen_state_dict
+            }, save_name)
+
+        # train iter
+        iter_start_time = time.time()
+        for iter_idx, data in enumerate(train_data_loader, 0):
+            global_iter += 1
+            in_text, text_lengths, target_vec, in_audio, aux_info, cluster_id = data
+            batch_size = target_vec.size(0)
+
+            in_text = in_text.to(device)
+            # in_audio = in_audio.to(device)
+            target_vec = target_vec.to(device)
+            cluster_id = torch.cat(cluster_id).to(device)
+            # train
+            loss = train_iter_c2g_seq2seq(args, epoch, cluster_id, target_vec, generator, gen_optimizer)
+
+            # loss values
+            for loss_meter in loss_meters:
+                name = loss_meter.name
+                if name in loss:
+                    loss_meter.update(loss[name], batch_size)
+
+            # print training status
+            if (iter_idx + 1) % print_interval == 0:
+                print_summary = 'EP {} ({:3d}) | {:>8s}, {:.0f} samples/s | '.format(
+                    epoch, iter_idx + 1, utils.train_utils.time_since(start),
+                    batch_size / (time.time() - iter_start_time))
+                for loss_meter in loss_meters:
+                    if loss_meter.count > 0:
+                        print_summary += '{}: {:.3f}, '.format(loss_meter.name, loss_meter.avg)
+                        loss_meter.reset()
+                logging.info(print_summary)
+
+            iter_start_time = time.time()
+
+
+def evaluate_testset(test_data_loader, generator, loss_fn, args):
+    # to evaluation mode
+    generator.train(False)
+
+    losses = AverageMeter('loss')
+    start = time.time()
+
+    with torch.no_grad():
+        for iter_idx, data in enumerate(test_data_loader, 0):
+            # in_text, text_lengths, target_vec, in_audio, aux_info, cluster_di = data
+            in_text, text_lengths, target_vec, audio, aux_info, cluster_id = data
+
+            batch_size = target_vec.size(0)
+
+            in_text = in_text.to(device)
+            target = target_vec.to(device)
+            # Todo: why the collate doesn't work for cluster_id?
+            cluster_id = torch.cat(cluster_id).to(device)
+            out_poses = generator(cluster_id, target)
+            loss = loss_fn(out_poses, target)
+            losses.update(loss.item(), batch_size)
+
+    # back to training mode
+    generator.train(True)
+
+    # print
+    elapsed_time = time.time() - start
+    logging.info('[VAL] loss: {:.3f} / {:.1f}s'.format(losses.avg, elapsed_time))
+
+    return losses.avg
+
+
+def main(config):
+    args = config['args']
+
+    trial_id = None
+
+    # random seed
+    if args.random_seed >= 0:
+        torch.manual_seed(args.random_seed)
+        torch.cuda.manual_seed_all(args.random_seed)
+        np.random.seed(args.random_seed)
+        random.seed(args.random_seed)
+        os.environ['PYTHONHASHSEED'] = str(args.random_seed)
+
+    # set logger
+    utils.train_utils.set_logger(args.model_save_path, os.path.basename(__file__).replace('.py', '.log'))
+
+    logging.info("PyTorch version: {}".format(torch.__version__))
+    logging.info("CUDA version: {}".format(torch.version.cuda))
+    logging.info(pprint.pformat(vars(args)))
+
+    # dataset
+    train_dataset = TrinityDataset_with_cluster(args, args.train_data_path[0],
+                                   n_poses=args.n_poses,
+                                   subdivision_stride=args.subdivision_stride,
+                                   pose_resampling_fps=args.motion_resampling_framerate,
+                                   data_mean=args.data_mean, data_std=args.data_std)
+    train_loader = DataLoader(dataset=train_dataset, batch_size=args.batch_size,
+                              shuffle=True, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                              collate_fn=word_seq_collate_fn
+                              )
+
+    val_dataset = TrinityDataset_with_cluster(args, args.val_data_path[0],
+                                 n_poses=args.n_poses,
+                                 subdivision_stride=args.subdivision_stride,
+                                 pose_resampling_fps=args.motion_resampling_framerate,
+                                 data_mean=args.data_mean, data_std=args.data_std)
+    test_loader = DataLoader(dataset=val_dataset, batch_size=args.batch_size,
+                             shuffle=False, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                             collate_fn=word_seq_collate_fn
+                             )
+
+    # build vocab
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    lang_model = build_vocab('words', [train_dataset, val_dataset], vocab_cache_path, args.wordembed_path,
+                             args.wordembed_dim)
+    train_dataset.set_lang_model(lang_model)
+    val_dataset.set_lang_model(lang_model)
+
+    # train
+    train_epochs(args, train_loader, test_loader, lang_model,
+                 pose_dim=82, trial_id=trial_id)
+
+
+if __name__ == '__main__':
+    _args = parse_args()
+    main({'args': _args})
diff --git a/scripts/train_eval/__pycache__/train_seq2seq.cpython-37.pyc b/scripts/train_eval/__pycache__/train_seq2seq.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f227413711f344ae94f30139246a87a7a82ec313
GIT binary patch
literal 8352
zcmb_hOOGShb*?vy#p0vc?DuHYQf%2RkGef=S$33xUh}dcP218;ET<x)Qq5c4O*UC%
z-zv_dQw$LFIMybP5@Ztx0oYmyh!=*p@g`Yik#%4MwMq~m!2AI(v`F%udyCbao*Bi9
z6j8Ts-Fxoq)H&aI@SSSaQE>fi<>$NaUs06*q?hSspz;>(c!Gi{OzkP2e5#%*-nG82
zDXP*pJX56}t!MR1UP;u|p550y9d+F+GbM1C!OVw>S78<_;pyH~n9a%$6;=-QZuLa>
zYRq93^wgQls(3b7jn(m-V+}TkXOlJAJf8Ec#TM{vu|>9o=K@o2D;vv?uuElUL(Qyr
zZ-qe=-)QU9f{Tt=!cFh-Kco0qnJA~qSWnc6_OX(xr`p&^ja2LEzpIRmpD24OQ<?VA
zP7J1htaMGLuP7(lHRVJdn~60k;kHxrR6SKweN7o#sfDp+jJ43?e5~N8`l&u9x>8y?
zRV#|1vr`*%70}sO&011Ydr29W6PKBX8vj*Noz$3>8k72|l3Jp)soc{Sl~WarECnuT
z^=~PGo0f58d#A19G<0dW%ed!p$8Vw7S5oCi-yW+84vR0N{8&rro8h^K!_2g)%XvSw
zFXM=RsT~=ZT-$j>lY^gnup4h^nYQ`$OpS=YgHbXZC4aAc{*$fVpdIvHiQ7C#+6P<1
z?w}t&2#&Y-Ant_zei$bs9{Rgs)IR73{BWz?8$==7y4oAXNyt0V{#GBgt*e7qZV$t-
zec<2BhujRK5Oa10(H17}43l^(;Xx<z!@EH*ug2lgHtx;ggU_8Sy-u982mRq2f&;`3
zOF4eNjoAl%zc+~E%`0NB@f)a2X*o&6BO-ppP#k}4ef_IMyN=EE<Bj#+<@GObesk~A
zdV3HhdG&6<MU4yYG%_<9^}}AK-pES5%VH(3MsxUhyBBu$50cE}L9`!cW_%C~@kj=|
zeURz#sE?*3XdikFF@P4C4l$wu?|XI8juofPmT2f;slom}*q50s+#T&_MkfjT8xAKa
z&#WZi`(ct9c*I^AJnM!1FiLoxdJWdu+w;m|F`A7pP^(Rg&}(LhMjMSAs+f<HTxFFR
zR#wmuUqqp(w(hEqeo<|qtf>vv(Okn;*HuHcHU2y(+O#FQXciOqRow9hC?MicLRCr4
zR7>^LV9JRBl~L0YltVeup(OT1BZ@@t8eMZ@rRJn`3LHAsFbXP2eM|#Y`Bdo;2IxDF
zXbDbx+i<e-amWY$J`b4KN@nkZ+4lm@coPJf7DbskN;<tbtG9cdp`i4M@qXqE10M84
z$N@M4Ssn5fnzPK;Xrvhq2l#{~$YT=TPcjFUUKs5s2XW>~?)$?*9LAX*;X{~%VbbYm
zC8;;D*d8Yi%R=zCDa9M#k^8w|e-#vQ4TYlFdJT-W^%k*p3oUJGo?axybj4pn5h?`E
zo_2;}p3Zb;04RZMhT@rEl$n)3xOMxz>+j)<7Z^eUO}aqS#mguX<wTjN;tOaK9iXK6
z_(~YA?qCKp9t}gD*}mV2I*IS|-@%ZqbnCX5HPicX2(U2Xu(yY2+4tMM0BYf%9hz@r
z#P88+WNH-EQIDI&wl=5p7BEixB3@m^9lwMk!R}IJqT#6n-P9$(q^N7MeyW|4RA@Uw
z&o>PD*^T&!C;U~6&vdBKhR(l9bz{gwmRUhN83n!bU#GM;;3PP2($d5uMLnK7+iKps
zVG%bX$c+yP^8NmRje1mfeg6ov${ulQPRGuwIaAtUuQz2(u6eJaUuYixI%==u7EH2L
z*AOjj3r%!2a2H&;i*E=mk$R<c!4nvkOR3V;m?mm4awqCX`q&zm#&&cEJ(7lM;2LP|
zkRb$(DQU*#u@lh@Fo@{AU(nsp>1ZyZA(Mu2FoDq4i3#&sNh>gn`iVAni*^2$>Emiz
z<^KZxMc=8~Es<aF=PGG4=t`-}tTou7aqZ~eQoO@J)*>Ty$F=FopQ!vLR!a1VO*b>r
zG7M>%Fko3por%NjQ?h9Wef^d)t|t{%PU|Ox8SpUDdQ15h=1gl7x4;${UTJojBe2Ds
zG^lU7Ypl4(Dych}Bb-U?J?*xF_nLf1opi$8$&5RE1bn{{^y2V_lsQ;W*6DK>A88*q
zF5{m#;iqT53o8U$g1W@3CVv6-tQL%t0SxAV0SWFNW%bA2gHQ;u&^7)IqPHQ#{*VuL
zIa$8U=)?EP^zi;L(}#n5nK^`+P56tn48a5cG8Lpf{3}!lSZfGa;{-TfB^q?%2NJ&c
z*NEg26|YdSi6SfAJ(8B93iTiM!uy@%0c{kvX&+VuiXV>1Ktb2{qRi%DTUtf)=I!s@
zxFgO=n@0D$?_E2uXtfy_y!Q6>cd%egssIQyTCguI@oNAO9SuL&8ADwJI69i4x#}gf
zxacF8bk&B&-^a*8Q>u6=H09TLc|ucOA{i!${m7U~e+0?M+cauRG?HmypPZ?D@XweA
zSv2TodM=O90LY^~u^@>gk9sbT6=p)t)zm#S_-B}*#4PC5XQ`7`Sg8ePG<BC1Je97!
zMw044rXkzH3Oe1&#D%0*VFk@xQft)5Y*4{f6jrdj!{5T`a(Vmre-x4^gicD^+tm5T
ztzX>ykK>1LO4(}&+2hyHBTS_fvK#bzlZx-4@K$FfagT@Lap>oQnpwgk#!}A65IkGh
zs%OHMRXPzo$uAH#)#tZpw)dzYv(CSdBC8$teA1r87wevvqKh+9^!AK8<%abxrid?~
zAj9gYEv;s_x}#sR4YjGR=JLaTfc`eg3tdEA$W8+m;0f9JppX-T@NiriyO0XlO}Zf!
zE>Oyy7(yy46Z4!@R3DQwj8827CrOE!snxX!yHH1H&@Fczfn-zM{smS=_#q^=Hm;8w
z<2ke-=nyT<@%*?oUWhIfRN5F+1%);S1y;&1dnv8wD`}HTT4(l(f)X*o9&i*WY5@f`
z&;d(l=;j68d_gy#AT-JKW03-yw4Tl*yjl|DJ&Xqe7bmKuOGVvHZCOukrXOnK<#Z`s
zUQt*Tx`5gh1$?ZhOW4Z_bfb|j!ZNH*H4uCI6|0Tcpgn7c2EC=LX_W!%sU_E^IiXLh
z>FT5@^hxW^cUzMM=+h$fsgdi`lIXKO(YS%#&ZUbv_nYZTUYn=&sm@xMYdP1tg=cDA
z<_a(09|j#BUp{WZCER480QgP#jllyrna9n`o9n$Gftx<OyeR^O*F}VJ+}wVm<G6l~
z@_4z8fGZ09xYO_S0uDsZ=g2B@?Ow1O_8$EYu16FN@Bi#?==z5@1tMfJU>QY%DrBBZ
zej#>-F(Er{3a}P$c^vY(M2(dh)F6+VKMok2d`SLXD&|mRrToyDJw0`19f<8o%pq4d
zn|tCU!mfF9Pi%ry5b@gp&V@6;;kRkbIji;|+C0f8@rkzi4~dG59lt}x83Gabcuwxe
z`@_qgBfmsUb)!G>FMDK<DVRcELAO0aw=L+l3%YIohcu~;bNdV~0W5eBAA0WDc|#_V
z@Aqjh<Q~d`;3#|XSx6THAZ2{~cc2pim!jK-1BBazy);x;t-;}L68fn%jouek3)XYd
zbPbnk;%AGP4pANKtfQ{#!nUH<!OvBBfVm6%N?RjGUijxbs1z8PTTu9hYGFNrQRkvN
zV<x&Yi|DR26WvMsC8IkjdlX_y84v>a#g@lD69lg+El-|_I4=_7+Y57Zj^B$j{BCdW
zPkH=b*hUemImXln;k#&Wiwqkov_qOs@aN~`sh7`r>Mv7*h+<*H#3jVT)<+syl4!AL
zw`SX?=*J(DjY%Du-(ebMdCqb@srmGFpt2kN2KL1mR}&L9z=D;+o3JlPT#<zu*AgqO
zogn!ndTX*5tJ&~=%L)9EQ<XdsvJbFN^{xR=8#%vc?L~Mqm^pR21}m*780n<e?Edz&
z6_HTO*S8WAX#Ngnx2JnUy-ef9J{|Jn$sd})4;nWRRm{H6VTX0{<>4K{cSkRfU&JHs
zWM1Gr`h^X!g$<D2e|aKofbjUI=<e=ta+`!Jm1*rZ{{y^rF5x2{8?WG!KK)W#CboQ;
z1BiWOyZnPeBC%whCJ@3afzw;2<iWNd_IE?Zkj)|eZ*xjI2q?4>K?~Pez~>p7W+fiV
zR8lFQKhuy?BJs*hIQy@?n(6J~D6?3I?9zeAPz@i*@Vs#qsk!(%B7(v20ik?m?)3%%
zPzRa0{r@9OXrAd0-=M?1jKZ^#{0K&@GZ0GTA(bMMds7~GwVU6+^9}#{^pNMm35l@8
zl}sDNnR=9|_cAre)ZOz?w=$!8f^h}kKF8?z+bDo=@aLAbMU<~=7BCChDHSm@qK2Be
zsx}cBIA|wqYv8x+<|&l~V+%bcOeuW5|Hday;B66B!7D?i;Z*sH79<=g-y&;O`4BPS
z`^pE(4@sxt<DuV{Po}~TR1x!*X6S2}$4qPFSHbUxj|7=)pv_7fXrm~Y=B|j0*c{qR
z=^WadiOoMv%E)*%r57mcQV-I(w24{f#Vqrnt|EG_!bfVQtw}}fs4OF1c%iwzP{c4^
zNUM*%FCs!;N|&H@e+`6&?u?i3|5Y-VE}f{?lpp;Wys!L(|DPuV7k=ygf1(~a9~S(5
z>^rRHoLd!~TN7Mf11~OM{<ZW1`173L&vR(Im|o2JV&{CJGyeu>Zl=qWZci7|MT+z{
zj2+>LWzJr32!$lCh<lKcjrp@!JS)9(Rpz})QJ!*q0VG)&8VXOUa~KNeD03(!nKwM{
z9!cF3F?k0gd6$Yqn!|yQl!O3>cETy!lOZh}Fp7Wj){W<%rkLl{aX8?VPWQ}x9}cKz
zGVv(w`yx5%l~`V}ciR$mJWHJSf#))xhe6K|IUjI(6YcWN5j}8uI*FIpi9dNgr;&Uk
z8a~4QJ;_MfX)&1{sN{=~+k^jz<Qn3|9YuVA{1|0nW6x$jWpPKtOg+R|!pMK%mFXe(
zAgjS{FO;ov5u8#~&O^#`&x92J=G!~3LG|MUY*^796R?8BwoM5KI85aJIN~WG25&m`
z+8SU5&XuLvaH<3~(&A^-G>YhIx~<iw^Uz3ulGEU0tW~&E<TDpA^zux|3B|L7>8FI;
z71U&M?+V-of(^J>hsKHikx21PL(zi%K+q$=;s@vGk$fN96t)UlqWEbs_fI9Q2r7vR
z0ct_@%VNe~7E~I|mFQ4MnJTGHXM&1-%8UdFLb?&&ItYuMi6z3~GK{_@k|l5{MDnhN
zL?7rRMd~1pVj`)M(^0w?3BDRqzxE>j7C6)}`yVCvQ&Bi9HbTuRfg@UwtjlYZdQrM%
zS*|H3hUk|g9E?8(H4>DMj|0c+7*Q9Y<)>*CiNu-+E$ew`Sr(yXRq`IQ{914a-$&Gq
zZl&vX5ey3#rHWHmS_-SLDS&6JMKFwA7~Mv94&Y;uYf^-xEdkvW?$c}<U!_w`7DOV_
zr5QysvS`bbk#iEVL>@>uE>B4YO`F~nVCJ>_r(9<#m*dKH`p;us{3*x32iYuQ$j&EF
z6ych0EK0AOJlro%Cdsf+L~sOy`2Pt>MT&%xH!tjZX30O$Bo@9%lYX&-KR1JePl@!r
z8dd>p7HOlWgm|9JT;4t6@6viA2B4^qQ}pGT*qeXvJdC_B10%EPrXRdNjSv2opyWH4
z0U0ONuv+Q`cqGfX5r3(aeR8!+>X#6a*#ayU@dO|s!zrRJ%4))YX&Ay=aa_Hn@jt~}
zVD}Op!U}ow;%~*xDB9c`MQwt5{Eb2Eg55f65RPR%KJ0}7k2Z5xjRRcp38oWBPjiuG
z7U32B-;`Om#Ge`BzXy?475Om{0E#GwlM5vPOl)n2MIwSCAj|A4@*j~mzzro{M?qNF
bBzH_RfSUGmcC%^Qui9U+ziYSbD|Yz5VI|U4

literal 0
HcmV?d00001

diff --git a/scripts/train_eval/train_seq2seq.py b/scripts/train_eval/train_seq2seq.py
new file mode 100644
index 0000000..b38fd5f
--- /dev/null
+++ b/scripts/train_eval/train_seq2seq.py
@@ -0,0 +1,489 @@
+import logging
+import torch
+import torch.nn.functional as F
+from sklearn.cluster import KMeans
+import torch.nn as nn
+
+debug = False
+
+
+loss_i = 0
+def custom_loss(output, target, args):
+    n_element = output.numel()
+
+    # MSE
+    l1_loss = F.l1_loss(output, target)
+    l1_loss *= args.loss_l1_weight
+
+    # continuous motion
+    diff = [abs(output[:, n, :] - output[:, n-1, :]) for n in range(1, output.shape[1])]
+    cont_loss = torch.sum(torch.stack(diff)) / n_element
+    cont_loss *= args.loss_cont_weight
+
+    # motion variance
+    norm = torch.norm(output, 2, 1)
+    var_loss = -torch.sum(norm) / n_element
+    var_loss *= args.loss_var_weight
+
+    loss = l1_loss + cont_loss + var_loss
+
+    # inspect loss terms
+    global loss_i
+    if loss_i == 100:
+        logging.debug('  (loss terms) l1 %.5f, cont %.5f, var %.5f' % (l1_loss.item(), cont_loss.item(), var_loss.item()))
+        loss_i = 0
+    loss_i += 1
+
+    return loss
+
+
+def train_iter_seq2seq(args, epoch, in_text, in_lengths, target_poses, net, optim):
+    # zero gradients
+    optim.zero_grad()
+
+    # generation
+    outputs = net(in_text, in_lengths, target_poses, None)
+
+    # loss
+    loss = custom_loss(outputs, target_poses, args)
+    loss.backward()
+
+    # optimize
+    torch.nn.utils.clip_grad_norm_(net.parameters(), 5)
+    optim.step()
+
+    return {'loss': loss.item()}
+
+
+class RMSLELoss(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.mse = nn.MSELoss()
+
+    def forward(self, pred, actual):
+        return torch.sqrt(self.mse(torch.log(pred + 1), torch.log(actual + 1)))
+
+def train_iter_DAE(args, epoch, noisy_poses, target_poses, net, optim):
+    # zero gradients
+    optim.zero_grad()
+
+
+    # generation
+    if args.autoencoder_vq=='True' and args.autoencoder_vae == 'False':
+        outputs, vq_loss, perplexity_vq = net(noisy_poses)
+    elif args.autoencoder_vq == 'True' and args.autoencoder_vae == 'True':
+        outputs, vq_loss, perplexity_vq, logvar, meu = net(noisy_poses)
+    elif args.autoencoder_vq == 'False' and args.autoencoder_vae == 'True':
+        outputs, logvar, meu = net(noisy_poses)
+    else:
+        outputs = net(noisy_poses)
+
+    # loss
+    loss_fn = torch.nn.MSELoss()
+
+    rec_loss = loss_fn(outputs, target_poses)
+
+    if args.autoencoder_vq == 'True':
+        GSOFT = False
+        if GSOFT:
+            rec_loss = outputs.log_prob(target_poses).sum(dim=1).mean()
+
+
+            loss = vq_loss - rec_loss/100
+            # print("LOSSSSSSSS!", vq_loss, rec_loss)
+        else:
+            loss = rec_loss + vq_loss
+    else:
+        loss = rec_loss
+
+    if args.autoencoder_vae == 'True':
+        loss_KLD = -2.5 * torch.mean(torch.mean(1 + logvar - logvar.exp() - meu.pow(2), 1))
+        # loss_KLD = -0.5 * torch.mean(torch.mean(1 + logvar - logvar.exp(), 1))
+        print("Kista", loss_KLD)
+        loss += 5 * loss_KLD #0.11
+
+    loss.backward()
+
+    # optimize
+    torch.nn.utils.clip_grad_norm_(net.parameters(), 5)
+    optim.step()
+
+    if args.autoencoder_vq == 'True':
+        return {'loss': rec_loss.item()}, perplexity_vq
+    else:
+        return {'loss': loss.item()}
+
+
+
+def train_iter_Autoencoder_seq2seq(args, epoch, input_poses, target_poses, net, optim):
+    # zero gradients
+    optim.zero_grad()
+
+    # generation
+    if args.autoencoder_vae == 'True':
+        outputs, _, meu, logvar = net(input_poses, target_poses)
+    else:
+        outputs , _ = net(input_poses, target_poses)
+
+    # loss
+    #Todo: important: I removed custom loss and replaced it by ll to test
+    loss = custom_loss(outputs, target_poses, args)
+    # loss  = F.mse_loss(outputs, target_poses)
+
+    if args.autoencoder_vae == 'True':
+
+        # loss_KLD = 0.5 * torch.mean(logvar.exp()-logvar-1 + meu.pow(2))
+        loss_KLD = -0.5 * torch.mean(torch.mean( 1  + logvar - logvar.exp() - meu.pow(2), 1))
+        # if epoch%10==0:
+        #     print("____________________")
+        #     print("loss", loss)
+        #     print("loss_KLD", loss_KLD )
+        #     print("Epoch ratio",epoch, args.epochs, epoch/args.epochs)
+        #     print("____________________")
+
+        kl_start_epoch = 5
+        if epoch>kl_start_epoch and args.autoencoder_freeze_encoder == 'False':
+            loss += loss_KLD * 0.01 * (epoch-kl_start_epoch)/args.epochs
+            # print("!!!!!!!!!!!!!!!!!!!!!!!!!!")
+    loss.backward()
+
+    # optimize
+    torch.nn.utils.clip_grad_norm_(net.parameters(), 5)
+    optim.step()
+
+    return {'loss': loss.item()}
+
+
+def train_iter_Autoencoder_ssl_seq2seq(args, epoch, input_poses, target_poses, net, optim,
+                                       stack_pairs1, stackpairs2, stack_label):
+    # zero gradients
+    optim.zero_grad()
+
+    # generation
+    # Unlabeled
+    if args.autoencoder_vae == 'True':
+        outputs, _, meu, logvar = net(input_poses, target_poses)
+        loss_KLD = -0.5 * torch.mean(torch.mean(1 + logvar - logvar.exp() - meu.pow(2), 1))
+
+    else:
+        outputs, _ = net(input_poses, target_poses)
+
+    # labeled
+    if debug:
+        print("stack_pairs1", stack_pairs1.shape)
+        print("net.decoder.n_layers", net.decoder.n_layers)
+    if args.autoencoder_vae == 'True':
+        outputs_p1, latents_p1, mu_1, logvar_1 = net(stack_pairs1, stack_pairs1)
+        outputs_p2, latents_p2, mu_2, logvar_2 = net(stackpairs2, stackpairs2)
+    else:
+        outputs_p1, latents_p1 = net(stack_pairs1, stack_pairs1)
+        outputs_p2, latents_p2 = net(stackpairs2, stackpairs2)
+
+    if debug:
+        print("1. latentp1.shape:", latents_p1.shape)
+    latents_p1 = torch.hstack((latents_p1[0], latents_p1[1]))
+    if debug:
+        print("2. latentp1.shape:", latents_p1.shape)
+
+    latents_p2 = torch.hstack((latents_p2[0], latents_p2[1]))
+    # Normal loss
+    #Todo: important: I removed custom loss and replaced it by ll to test
+    # loss = custom_loss(outputs, target_poses, args)
+    # loss_unlabeled = F.mse_loss(outputs, target_poses)
+    loss_unlabeled = custom_loss(outputs, target_poses, args)
+
+    # Pairwise loss
+    # cos_dist = [torch.nn.functional.cosine_similarity(stack_pairs1[n, n, :], output[:, n - 1, :]) for n in range(1, output.shape[1])]
+    cos_dist = F.cosine_similarity(latents_p1, latents_p2)
+    if debug:
+        print("latentp1.shape:", latents_p1.shape)
+        print("cosine_similarity.shape:", cos_dist.shape)
+        print("stack_label", stack_label.shape)
+    mask = (stack_label == 1)
+    cos_dist[mask] = cos_dist[mask]*-1
+    loss_labeled = torch.sum(cos_dist)
+
+    loss = args.loss_label_weight + loss_unlabeled
+
+    if args.autoencoder_vae == 'True':
+        kl_start_epoch = 10
+        if epoch > kl_start_epoch:
+            loss += loss_KLD * 0.1 * (epoch - kl_start_epoch) / args.epochs
+
+
+    loss.backward()
+
+    # optimize
+    torch.nn.utils.clip_grad_norm_(net.parameters(), 5)
+    optim.step()
+    if debug:
+        print("loss_unlabeled", loss_unlabeled)
+        print("Loss_labeled", loss_labeled)
+        print("loss_label_weight:", args.loss_label_weight)
+    return {'loss': loss.item()}
+
+
+
+
+
+
+def train_iter_c2g_seq2seq(args, epoch, input_Cluster, target_poses, net, optim):
+    # zero gradients
+    optim.zero_grad()
+
+    # generation
+    outputs = net(input_Cluster, target_poses)
+
+    # loss
+    #Todo: important: I removed custom loss and replaced it by ll to test
+    loss = custom_loss(outputs, target_poses, args)
+    # loss  = F.mse_loss(outputs, target_poses)
+    loss.backward()
+
+    # optimize
+    torch.nn.utils.clip_grad_norm_(net.parameters(), 5)
+    optim.step()
+
+    return {'loss': loss.item()}
+
+
+def train_iter_text2embedding(args, epoch, in_text, in_lengths, in_audio, target_poses, cluster_targets,
+                              GPT3_Embedding, net, optim):
+    # zero gradients
+    optim.zero_grad()
+
+    # generation
+    if args.text2_embedding_discrete == 'False':
+        outputs, _ = net(in_text, in_lengths, in_audio, target_poses, GPT3_Embedding, None)
+    else:
+        outputs, _ = net(in_text, in_lengths, in_audio, cluster_targets, GPT3_Embedding, None)
+    # loss
+    # print(outputs.shape)
+    if args.text2_embedding_discrete=='False':
+        loss = F.mse_loss(outputs[:, 1:, :], target_poses[:, 1:, :])
+    else:
+        os = cluster_targets.shape
+        if debug:
+            print("cc", cluster_targets.shape)
+            q = cluster_targets.reshape(-1)
+            print(q)
+            print("cc", q.shape)
+            w = F.one_hot(q.to(torch.int64), 300)
+            print("----", w.shape, w)
+
+
+        # cluster_targets_one_hot = F.one_hot(cluster_targets.reshape(-1).to(torch.int64), 300)
+        # cluster_targets_one_hot = cluster_targets_one_hot.reshape(os[0], os[1], -1)
+        outputs = outputs[:, 1:, :]
+        outputs = outputs.reshape(-1, outputs.shape[2])
+        # cluster_targets.reshape(-1)
+
+        cluster_targets = cluster_targets[:,1:]
+        cluster_targets = cluster_targets.reshape(-1)
+
+        if debug:
+            print("cluster_targets_one_hot", cluster_targets.shape)
+            print("outputs", outputs.shape)
+        a = outputs.cpu().detach().numpy()
+        b = cluster_targets.cpu().detach().numpy()
+        loss = torch.nn.CrossEntropyLoss()(outputs.float(), cluster_targets.long())
+        # loss = torch.nn.NLLLoss()(outputs.float(), cluster_targets.long())
+
+        # cluster_targets_one_hot = F.one_hot(cluster_targets.reshape(-1).to(torch.int64), 514)
+        # loss = F.l1_loss(outputs, cluster_targets_one_hot)
+
+    # loss2 = torch.mean(torch.mean(loss, dim=2)/cluster_portion)
+
+
+
+
+
+    loss.backward()
+
+    # optimize
+    torch.nn.utils.clip_grad_norm_(net.parameters(), 5)
+    optim.step()
+
+    return {'loss': loss.item()}
+
+
+def train_iter_text2embedding_GAN(args, epoch, in_text, in_lengths, target_poses, cluster_portion,
+                                  g_net, d_net, g_optim, d_optim):
+    # zero gradients
+    bce_loss = torch.nn.BCELoss()
+
+    # generation
+    # 1. * Generate fake data
+
+    with torch.no_grad():
+        fake_y = g_net(in_text, in_lengths,  target_poses, None)
+
+    # 2. * Train Discriminator
+
+    d_optim.zero_grad()
+    g_optim.zero_grad()
+    d_real_error = 0
+    d_fake_error = 0
+
+    real_logit = d_net(in_text, in_lengths, target_poses, None )
+    real_label = torch.ones_like(real_logit)
+    real_error = bce_loss(real_logit, real_label)
+    d_real_error = torch.mean(real_error)
+
+    fake_logit = d_net(in_text, in_lengths, fake_y, None )
+    fake_label = torch.zeros_like(fake_logit)
+    if debug:
+        print("fake_label", fake_label.shape)
+        print("fake_label", fake_label)
+    fake_error = bce_loss(fake_logit, fake_label)
+    d_fake_error = torch.mean(fake_error)
+
+
+    d_loss = d_real_error + d_fake_error
+    d_loss.backward()
+    d_optim.step()
+
+    d_real_loss = (d_real_error.cpu().detach().numpy())
+    d_fake_loss = (d_fake_error.cpu().detach().numpy())
+
+    # 2. * Unrolling step
+    unroll_steps = 10
+    if unroll_steps:
+        # * Unroll D
+        d_backup = d_net.state_dict()
+        for k in range(unroll_steps):
+            # * Train D
+            d_optim.zero_grad()
+            d_real_error = 0
+            d_fake_error = 0
+
+
+            real_logit = d_net(in_text, in_lengths, target_poses, None )
+            real_label = torch.ones_like(real_logit)
+            real_error = bce_loss(real_logit, real_label)
+            d_real_error = torch.mean(real_error)
+
+            fake_logit = d_net(in_text, in_lengths, fake_y, None)
+            fake_label = torch.zeros_like(fake_logit)
+            fake_error = bce_loss(fake_logit, fake_label)
+            d_fake_error = torch.mean(fake_error)
+
+            d_loss = d_real_error + d_fake_error
+            d_loss.backward()
+            d_optim.step()
+
+    # * Train G
+    g_optim.zero_grad()
+
+    gen_y = g_net(in_text, in_lengths, target_poses, None )
+
+    gen_logit = d_net(in_text, in_lengths, gen_y, None)
+    gen_lable = torch.ones_like(gen_logit)
+    gen_error = bce_loss(gen_logit, gen_lable)
+    g_error = torch.mean(gen_error)
+
+
+    g_error.backward()
+    g_optim.step()
+
+    if unroll_steps:
+        d_net.load_state_dict(d_backup)
+
+    # Todo: is it necessary
+    # torch.nn.utils.clip_grad_norm_(g_net.parameters(), 5)
+    # torch.nn.utils.clip_grad_norm_(d_net.parameters(), 5)
+
+    # # loss
+    # # print(outputs.shape)
+    # loss = F.mse_loss(outputs, target_poses)
+    # # loss2 = torch.mean(torch.mean(loss, dim=2)/cluster_portion)
+    # loss.backward()
+    # # optimize
+    # torch.nn.utils.clip_grad_norm_(net.parameters(), 5)
+    # optim.step()
+
+    # {'loss': loss.item()},
+    return d_real_loss, d_fake_loss
+
+
+
+
+def train_iter_Autoencoder_VQ_seq2seq(args, epoch, input_poses, target_poses, net, optim):
+    # zero gradients
+    optim.zero_grad()
+    vq_start_epoch = 0
+
+
+    # net.vq_layer.embedding_grad(epoch % 3 == 0)
+
+    # generation
+    if args.autoencoder_vq == 'True' and args.autoencoder_vae == 'True':
+        outputs, _, meu, logvar, loss_vq, perplexity_vq = net(input_poses, target_poses, epoch > vq_start_epoch)
+    if args.autoencoder_vq == 'True' and args.autoencoder_vae == 'False':
+        outputs, _, loss_vq, perplexity_vq = net(input_poses, target_poses, epoch > vq_start_epoch)
+    if args.autoencoder_vq == 'False' and args.autoencoder_vae == 'True':
+            outputs, _, meu, logvar = net(input_poses, target_poses)
+    if args.autoencoder_vq == 'False' and args.autoencoder_vae == 'False':
+        outputs , _ = net(input_poses, target_poses)
+
+    # loss
+    #Todo: important: I removed custom loss and replaced it by ll to test
+    loss = custom_loss(outputs, target_poses, args)
+    # loss  = F.mse_loss(outputs, target_poses)
+
+    if args.autoencoder_vae == 'True':
+        # loss_KLD = 0.5 * torch.mean(logvar.exp()-logvar-1 + meu.pow(2))
+        loss_KLD = -0.5 * torch.mean(torch.mean( 1  + logvar - logvar.exp() - meu.pow(2), 1))
+        loss_KLD = 0.5 * torch.mean(logvar.exp() - logvar - 1 + meu.pow(2))
+        if epoch%10==0:
+            print("____________________")
+            print("loss", loss)
+            print("loss_KLD", loss_KLD )
+            print("Epoch ratio",epoch, args.epochs, epoch/args.epochs)
+            print("____________________")
+        if debug:
+            print('rec_loss', loss)
+            # print("loss_vqn", loss_vq)
+            print("loss_KLD", loss_KLD)
+
+        kl_start_epoch = 0
+        if epoch>kl_start_epoch:
+            loss += loss_KLD * 0.1 * (epoch-kl_start_epoch)/args.epochs
+
+    if epoch > vq_start_epoch:
+        if args.autoencoder_vq == 'True':
+            emb_w = net.vq_layer._embedding.weight.detach()
+            # mycdist = torch.cdist(emb_w, emb_w).mean()
+            # print("________________________________________________\n loss:",
+            #       loss.data, "  vq_los", loss_vq.data, "mycdist:", mycdist.data)
+
+            loss = (loss) + 1 * loss_vq/400 #+ -1*torch.log(mycdist)
+
+
+
+    loss.backward()
+
+    # optimize
+    torch.nn.utils.clip_grad_norm_(net.parameters(), 5)
+    optim.step()
+
+    # if epoch % 1 == 0:
+    #     print("____________________")
+    #     print("loss", loss)
+    #     # print("loss_KLD", loss_KLD)
+    #     print("Loss:", loss, "-------Loss_VQ", loss_vq)
+    #     print("____________________")
+
+
+    if args.autoencoder_vq == 'True':
+        # return {'loss': loss.item()}, loss_vq
+        # Todo: it should be perplexity_vq not loss_vq
+        return {'loss': loss.item()}, perplexity_vq.detach()
+    else:
+        return {'loss': loss.item()}
+
+
+
+
+
+
diff --git a/scripts/train_gan.py b/scripts/train_gan.py
new file mode 100644
index 0000000..523b95d
--- /dev/null
+++ b/scripts/train_gan.py
@@ -0,0 +1,240 @@
+import datetime
+import pickle
+import pprint
+import random
+import time
+import sys
+import numpy as np
+
+from torch.utils.data import DataLoader
+import torch
+[sys.path.append(i) for i in ['.', '..']]
+
+from model import vocab
+from model.seq2seq_net import Seq2SeqNet
+from model.text2embedding_model import text2embedding_model
+from model.text2embedding_GAN_model import text2embedding_model_gan
+from train_eval.train_seq2seq import train_iter_seq2seq, train_iter_c2g_seq2seq, train_iter_text2embedding, train_iter_text2embedding_GAN
+from utils.average_meter import AverageMeter
+from utils.vocab_utils import build_vocab
+
+from config.parse_args import parse_args
+
+from torch import optim
+
+from data_loader.lmdb_data_loader import *
+import utils.train_utils
+
+import matplotlib.pyplot as plt
+
+
+
+
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+debug = False
+
+
+
+
+def init_model(args, lang_model, pose_dim, _device):
+    n_frames = args.n_poses
+    gen_disc = text2embedding_model_gan(args, pose_dim, n_frames, lang_model.n_words, args.wordembed_dim,
+                           lang_model.word_embedding_weights).to(_device)
+
+
+
+
+
+
+    bce_loss = torch.nn.BCELoss()
+
+
+    loss_fn = torch.nn.MSELoss()
+
+    return gen_disc, bce_loss
+
+def sample_noise(batch_size, dim):
+    return np.random.normal(0, 1, (batch_size, dim))
+
+
+
+def train_epochs(args, train_data_loader, test_data_loader, lang_model, pose_dim, trial_id=None):
+    start = time.time()
+    loss_meters = [AverageMeter('loss'), AverageMeter('var_loss')]
+
+    # interval params
+    print_interval = int(len(train_data_loader) / 5)
+    save_model_epoch_interval = 20
+
+    # init model
+    gen_disc, bce_loss = init_model(args, lang_model, pose_dim, device)
+
+    # define optimizers
+    gen_optimizer = optim.Adam(gen_disc.generator.parameters(), lr=args.learning_rate, betas=(0.5, 0.999))
+    disc_optimizer = optim.Adam(gen_disc.discriminator.parameters(), lr=args.learning_rate, betas=(0.5, 0.999))
+    # training
+    global_iter = 0
+    d_real_loss = []
+    d_fake_loss = []
+    for epoch in range(1, args.epochs+1):
+        # evaluate the test set
+        val_metrics = evaluate_testset(test_data_loader, gen_disc, torch.nn.MSELoss(), args)
+
+        # save model
+        if epoch % save_model_epoch_interval == 0 and epoch > 0:
+            try:  # multi gpu
+                gen_state_dict = gen_disc.module.state_dict()
+            except AttributeError:  # single gpu
+                gen_state_dict = gen_disc.state_dict()
+
+            save_name = '{}/{}_checkpoint_{:03d}.bin'.format(args.model_save_path, args.name, epoch)
+
+            utils.train_utils.save_checkpoint({
+                'args': args, 'epoch': epoch, 'lang_model': lang_model,
+                'pose_dim': pose_dim, 'gen_dict': gen_state_dict
+            }, save_name)
+
+        # train iter
+        iter_start_time = time.time()
+        for iter_idx, data in enumerate(train_data_loader, 0):
+            global_iter += 1
+
+            in_text, text_lengths, target_vec, in_audio, aux_info, sentence_leve_latents, c_portion = data
+            batch_size = target_vec.size(0)
+
+            in_text = in_text.to(device)
+            # in_audio = in_audio.to(device)
+            # target_vec = target_vec.to(device)
+            target_vec = sentence_leve_latents.to(device)
+            c_portion = c_portion.to(device)
+            # train
+            loss = train_iter_text2embedding_GAN(args, epoch, in_text, text_lengths,
+                                                 target_vec, c_portion,
+                                                 gen_disc.generator, gen_disc.discriminator,
+                                                 gen_optimizer, disc_optimizer)
+            d_real_loss.append(loss[0])
+            d_fake_loss.append(loss[1])
+
+            # loss values
+            for loss_meter in loss_meters:
+                name = loss_meter.name
+                if name in loss:
+                    loss_meter.update(loss[name], batch_size)
+
+            # print training status
+            if (iter_idx + 1) % print_interval == 0:
+                print_summary = 'EP {} ({:3d}) | {:>8s}, {:.0f} samples/s | '.format(
+                    epoch, iter_idx + 1, utils.train_utils.time_since(start),
+                    batch_size / (time.time() - iter_start_time))
+                for loss_meter in loss_meters:
+                    if loss_meter.count > 0:
+                        print_summary += '{}: {:.3f}, '.format(loss_meter.name, loss_meter.avg)
+                        loss_meter.reset()
+                print_summary += '\nABC,' + str(loss[0]) + ',' + str(loss[1])
+                logging.info(print_summary)
+
+            iter_start_time = time.time()
+
+    plt.plot(d_fake_loss)
+    plt.plot(d_real_loss)
+    plt.title("AAAAAAAAAAAAAAAAAAA")
+    plt.show()
+    losses = {'d_fake_loss': d_fake_loss,
+              'd_real_loss': d_real_loss}
+    pickle.dump(losses, open('losessss.plk', 'wb'))
+
+def evaluate_testset(test_data_loader, gen_disc, loss_fn, args):
+    # to evaluation mode
+    gen_disc.train(False)
+
+    losses = AverageMeter('loss')
+    start = time.time()
+
+    with torch.no_grad():
+        for iter_idx, data in enumerate(test_data_loader, 0):
+            in_text, text_lengths, target_vec, in_audio, aux_info, sentence_leve_latents, c_portion = data
+
+            batch_size = target_vec.size(0)
+
+            if debug:
+                print(sentence_leve_latents.shape)
+
+            in_text = in_text.to(device)
+            # target = target_vec.to(device)
+            target = sentence_leve_latents.to(device)
+
+
+            out_latents = gen_disc.generator(in_text, text_lengths, target, None)
+            if debug:
+                print("Compare", out_latents.shape, target.shape)
+            loss = loss_fn(out_latents, target)
+            losses.update(loss.item(), batch_size)
+
+    # back to training mode
+    gen_disc.train(True)
+
+    # print
+    elapsed_time = time.time() - start
+    logging.info('[VAL] loss: {:.3f} / {:.1f}s'.format(losses.avg, elapsed_time))
+
+    return losses.avg
+
+
+def main(config):
+    args = config['args']
+
+    trial_id = None
+
+    # random seed
+    if args.random_seed >= 0:
+        torch.manual_seed(args.random_seed)
+        torch.cuda.manual_seed_all(args.random_seed)
+        np.random.seed(args.random_seed)
+        random.seed(args.random_seed)
+        os.environ['PYTHONHASHSEED'] = str(args.random_seed)
+
+    # set logger
+    utils.train_utils.set_logger(args.model_save_path, os.path.basename(__file__).replace('.py', '.log'))
+
+    logging.info("PyTorch version: {}".format(torch.__version__))
+    logging.info("CUDA version: {}".format(torch.version.cuda))
+    logging.info(pprint.pformat(vars(args)))
+
+    # dataset
+    train_dataset = TrinityDataset_sentencelevel(args, args.train_data_path[0],
+                                   n_poses=args.n_poses,
+                                   subdivision_stride=args.subdivision_stride,
+                                   pose_resampling_fps=args.motion_resampling_framerate,
+                                   data_mean=args.data_mean, data_std=args.data_std)
+    train_loader = DataLoader(dataset=train_dataset, batch_size=args.batch_size,
+                              shuffle=True, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                              collate_fn=word_seq_collate_fn
+                              )
+
+    val_dataset = TrinityDataset_sentencelevel(args, args.val_data_path[0],
+                                 n_poses=args.n_poses,
+                                 subdivision_stride=args.subdivision_stride,
+                                 pose_resampling_fps=args.motion_resampling_framerate,
+                                 data_mean=args.data_mean, data_std=args.data_std)
+    test_loader = DataLoader(dataset=val_dataset, batch_size=args.batch_size,
+                             shuffle=False, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                             collate_fn=word_seq_collate_fn
+                             )
+    print("Data loaded successfully")
+    # build vocab
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    lang_model = build_vocab('words', [train_dataset, val_dataset], vocab_cache_path, args.wordembed_path,
+                             args.wordembed_dim)
+    train_dataset.set_lang_model(lang_model)
+    val_dataset.set_lang_model(lang_model)
+
+    # train
+    train_epochs(args, train_loader, test_loader, lang_model,
+                 pose_dim=15*9, trial_id=trial_id)
+
+
+if __name__ == '__main__':
+    _args = parse_args()
+    main({'args': _args})
diff --git a/scripts/train_stream_cluster.py b/scripts/train_stream_cluster.py
new file mode 100644
index 0000000..ef7418b
--- /dev/null
+++ b/scripts/train_stream_cluster.py
@@ -0,0 +1,189 @@
+import datetime
+import pprint
+import random
+import time
+import sys
+import numpy as np
+
+from torch.utils.data import DataLoader
+
+[sys.path.append(i) for i in ['.', '..']]
+
+from model import vocab
+from model.seq2seq_net import Seq2SeqNet
+from train_eval.train_seq2seq import train_iter_seq2seq
+from utils.average_meter import AverageMeter
+from utils.vocab_utils import build_vocab
+
+from config.parse_args import parse_args
+
+from torch import optim
+
+from data_loader.lmdb_data_loader import *
+import utils.train_utils
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+
+def init_model(args, lang_model, pose_dim, _device):
+    n_frames = args.n_poses
+    generator = Seq2SeqNet????(args, pose_dim, n_frames, lang_model.n_words, args.wordembed_dim,
+                           lang_model.word_embedding_weights).to(_device)
+    loss_fn = torch.nn.MSELoss()
+
+    return generator, loss_fn
+
+
+def train_epochs(args, train_data_loader, test_data_loader, lang_model, pose_dim, trial_id=None):
+    start = time.time()
+    loss_meters = [AverageMeter('loss'), AverageMeter('var_loss')]
+
+    # interval params
+    print_interval = int(len(train_data_loader) / 5)
+    save_model_epoch_interval = 20
+
+    # init model
+    generator, loss_fn = init_model(args, lang_model, pose_dim, device)
+
+    # define optimizers
+    gen_optimizer = optim.Adam(generator.parameters(), lr=args.learning_rate, betas=(0.5, 0.999))
+
+    # training
+    global_iter = 0
+    for epoch in range(1, args.epochs+1):
+        # evaluate the test set
+        val_metrics = evaluate_testset(test_data_loader, generator, loss_fn, args)
+
+        # save model
+        if epoch % save_model_epoch_interval == 0 and epoch > 0:
+            try:  # multi gpu
+                gen_state_dict = generator.module.state_dict()
+            except AttributeError:  # single gpu
+                gen_state_dict = generator.state_dict()
+
+            save_name = '{}/{}_checkpoint_{:03d}.bin'.format(args.model_save_path, args.name, epoch)
+
+            utils.train_utils.save_checkpoint({
+                'args': args, 'epoch': epoch, 'lang_model': lang_model,
+                'pose_dim': pose_dim, 'gen_dict': gen_state_dict
+            }, save_name)
+
+        # train iter
+        iter_start_time = time.time()
+        for iter_idx, data in enumerate(train_data_loader, 0):
+            global_iter += 1
+            in_text, text_lengths, target_vec, in_audio, aux_info, cluster???? = data
+            batch_size = target_vec.size(0)
+
+            in_text = in_text.to(device)
+            # in_audio = in_audio.to(device)
+            target_vec = target_vec.to(device)
+
+            # train
+            loss = train_iter_seq2seq(args, epoch, in_text, text_lengths, target_vec, generator, gen_optimizer)
+
+            # loss values
+            for loss_meter in loss_meters:
+                name = loss_meter.name
+                if name in loss:
+                    loss_meter.update(loss[name], batch_size)
+
+            # print training status
+            if (iter_idx + 1) % print_interval == 0:
+                print_summary = 'EP {} ({:3d}) | {:>8s}, {:.0f} samples/s | '.format(
+                    epoch, iter_idx + 1, utils.train_utils.time_since(start),
+                    batch_size / (time.time() - iter_start_time))
+                for loss_meter in loss_meters:
+                    if loss_meter.count > 0:
+                        print_summary += '{}: {:.3f}, '.format(loss_meter.name, loss_meter.avg)
+                        loss_meter.reset()
+                logging.info(print_summary)
+
+            iter_start_time = time.time()
+
+
+def evaluate_testset(test_data_loader, generator, loss_fn, args):
+    # to evaluation mode
+    generator.train(False)
+
+    losses = AverageMeter('loss')
+    start = time.time()
+
+    with torch.no_grad():
+        for iter_idx, data in enumerate(test_data_loader, 0):
+            in_text, text_lengths, target_vec, in_audio, aux_info = data
+            batch_size = target_vec.size(0)
+
+            in_text = in_text.to(device)
+            target = target_vec.to(device)
+
+            out_poses = generator(in_text, text_lengths, target, None)
+            loss = loss_fn(out_poses, target)
+            losses.update(loss.item(), batch_size)
+
+    # back to training mode
+    generator.train(True)
+
+    # print
+    elapsed_time = time.time() - start
+    logging.info('[VAL] loss: {:.3f} / {:.1f}s'.format(losses.avg, elapsed_time))
+
+    return losses.avg
+
+
+def main(config):
+    args = config['args']
+
+    trial_id = None
+
+    # random seed
+    if args.random_seed >= 0:
+        torch.manual_seed(args.random_seed)
+        torch.cuda.manual_seed_all(args.random_seed)
+        np.random.seed(args.random_seed)
+        random.seed(args.random_seed)
+        os.environ['PYTHONHASHSEED'] = str(args.random_seed)
+
+    # set logger
+    utils.train_utils.set_logger(args.model_save_path, os.path.basename(__file__).replace('.py', '.log'))
+
+    logging.info("PyTorch version: {}".format(torch.__version__))
+    logging.info("CUDA version: {}".format(torch.version.cuda))
+    logging.info(pprint.pformat(vars(args)))
+
+    # dataset
+    train_dataset = TrinityDataset_with_cluster(args, args.train_data_path[0],
+                                   n_poses=args.n_poses,
+                                   subdivision_stride=args.subdivision_stride,
+                                   pose_resampling_fps=args.motion_resampling_framerate,
+                                   data_mean=args.data_mean, data_std=args.data_std)
+    train_loader = DataLoader(dataset=train_dataset, batch_size=args.batch_size,
+                              shuffle=True, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                              collate_fn=word_seq_collate_fn
+                              )
+
+    val_dataset = TrinityDataset_with_cluster(args, args.val_data_path[0],
+                                 n_poses=args.n_poses,
+                                 subdivision_stride=args.subdivision_stride,
+                                 pose_resampling_fps=args.motion_resampling_framerate,
+                                 data_mean=args.data_mean, data_std=args.data_std)
+    test_loader = DataLoader(dataset=val_dataset, batch_size=args.batch_size,
+                             shuffle=False, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                             collate_fn=word_seq_collate_fn
+                             )
+
+    # build vocab
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    lang_model = build_vocab('words', [train_dataset, val_dataset], vocab_cache_path, args.wordembed_path,
+                             args.wordembed_dim)
+    train_dataset.set_lang_model(lang_model)
+    val_dataset.set_lang_model(lang_model)
+
+    # train
+    train_epochs(args, train_loader, test_loader, lang_model,
+                 pose_dim=82, trial_id=trial_id)
+
+
+if __name__ == '__main__':
+    _args = parse_args()
+    main({'args': _args})
diff --git a/scripts/train_text2embedding(stream_gesture_).py b/scripts/train_text2embedding(stream_gesture_).py
new file mode 100644
index 0000000..2dfc817
--- /dev/null
+++ b/scripts/train_text2embedding(stream_gesture_).py
@@ -0,0 +1,193 @@
+import datetime
+import pprint
+import random
+import time
+import sys
+import numpy as np
+
+from torch.utils.data import DataLoader
+import torch
+[sys.path.append(i) for i in ['.', '..']]
+
+from model import vocab
+from model.seq2seq_net import Seq2SeqNet
+from model.seq2seq_with_cluster_model import cluster2gesture_model
+from train_eval.train_seq2seq import train_iter_seq2seq, train_iter_c2g_seq2seq
+from utils.average_meter import AverageMeter
+from utils.vocab_utils import build_vocab
+
+from config.parse_args import parse_args
+
+from torch import optim
+
+from data_loader.lmdb_data_loader import *
+import utils.train_utils
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+
+def init_model(args, lang_model, pose_dim, _device):
+    n_frames = args.n_poses
+    generator = cluster2gesture_model(args, 40, args.hidden_size, args.hidden_size, args.rep_learning_dim, ).to(_device)
+    loss_fn = torch.nn.MSELoss()
+
+    return generator, loss_fn
+
+
+def train_epochs(args, train_data_loader, test_data_loader, lang_model, pose_dim, trial_id=None):
+    start = time.time()
+    loss_meters = [AverageMeter('loss'), AverageMeter('var_loss')]
+
+    # interval params
+    print_interval = int(len(train_data_loader) / 5)
+    save_model_epoch_interval = 20
+
+    # init model
+    generator, loss_fn = init_model(args, lang_model, pose_dim, device)
+
+    # define optimizers
+    gen_optimizer = optim.Adam(generator.parameters(), lr=args.learning_rate, betas=(0.5, 0.999))
+
+    # training
+    global_iter = 0
+    for epoch in range(1, args.epochs+1):
+        # evaluate the test set
+        val_metrics = evaluate_testset(test_data_loader, generator, loss_fn, args)
+
+        # save model
+        if epoch % save_model_epoch_interval == 0 and epoch > 0:
+            try:  # multi gpu
+                gen_state_dict = generator.module.state_dict()
+            except AttributeError:  # single gpu
+                gen_state_dict = generator.state_dict()
+
+            save_name = '{}/{}_checkpoint_{:03d}.bin'.format(args.model_save_path, args.name, epoch)
+
+            utils.train_utils.save_checkpoint({
+                'args': args, 'epoch': epoch, 'lang_model': lang_model,
+                'pose_dim': pose_dim, 'gen_dict': gen_state_dict
+            }, save_name)
+
+        # train iter
+        iter_start_time = time.time()
+        for iter_idx, data in enumerate(train_data_loader, 0):
+            global_iter += 1
+            in_text, text_lengths, target_vec, in_audio, aux_info, cluster_id = data
+            batch_size = target_vec.size(0)
+
+            in_text = in_text.to(device)
+            # in_audio = in_audio.to(device)
+            target_vec = target_vec.to(device)
+            cluster_id = torch.cat(cluster_id).to(device)
+            # train
+            loss = train_iter_c2g_seq2seq(args, epoch, cluster_id, target_vec, generator, gen_optimizer)
+
+            # loss values
+            for loss_meter in loss_meters:
+                name = loss_meter.name
+                if name in loss:
+                    loss_meter.update(loss[name], batch_size)
+
+            # print training status
+            if (iter_idx + 1) % print_interval == 0:
+                print_summary = 'EP {} ({:3d}) | {:>8s}, {:.0f} samples/s | '.format(
+                    epoch, iter_idx + 1, utils.train_utils.time_since(start),
+                    batch_size / (time.time() - iter_start_time))
+                for loss_meter in loss_meters:
+                    if loss_meter.count > 0:
+                        print_summary += '{}: {:.3f}, '.format(loss_meter.name, loss_meter.avg)
+                        loss_meter.reset()
+                logging.info(print_summary)
+
+            iter_start_time = time.time()
+
+
+def evaluate_testset(test_data_loader, generator, loss_fn, args):
+    # to evaluation mode
+    generator.train(False)
+
+    losses = AverageMeter('loss')
+    start = time.time()
+
+    with torch.no_grad():
+        for iter_idx, data in enumerate(test_data_loader, 0):
+            # in_text, text_lengths, target_vec, in_audio, aux_info, cluster_di = data
+            in_text, text_lengths, target_vec, audio, aux_info, cluster_id = data
+
+            batch_size = target_vec.size(0)
+
+            in_text = in_text.to(device)
+            target = target_vec.to(device)
+            # Todo: why the collate doesn't work for cluster_id?
+            cluster_id = torch.cat(cluster_id).to(device)
+            out_poses = generator(cluster_id, target)
+            loss = loss_fn(out_poses, target)
+            losses.update(loss.item(), batch_size)
+
+    # back to training mode
+    generator.train(True)
+
+    # print
+    elapsed_time = time.time() - start
+    logging.info('[VAL] loss: {:.3f} / {:.1f}s'.format(losses.avg, elapsed_time))
+
+    return losses.avg
+
+
+def main(config):
+    args = config['args']
+
+    trial_id = None
+
+    # random seed
+    if args.random_seed >= 0:
+        torch.manual_seed(args.random_seed)
+        torch.cuda.manual_seed_all(args.random_seed)
+        np.random.seed(args.random_seed)
+        random.seed(args.random_seed)
+        os.environ['PYTHONHASHSEED'] = str(args.random_seed)
+
+    # set logger
+    utils.train_utils.set_logger(args.model_save_path, os.path.basename(__file__).replace('.py', '.log'))
+
+    logging.info("PyTorch version: {}".format(torch.__version__))
+    logging.info("CUDA version: {}".format(torch.version.cuda))
+    logging.info(pprint.pformat(vars(args)))
+
+    # dataset
+    train_dataset = TrinityDataset_sentencelevel(args, args.train_data_path[0],
+                                   n_poses=args.n_poses,
+                                   subdivision_stride=args.subdivision_stride,
+                                   pose_resampling_fps=args.motion_resampling_framerate,
+                                   data_mean=args.data_mean, data_std=args.data_std)
+    train_loader = DataLoader(dataset=train_dataset, batch_size=args.batch_size,
+                              shuffle=True, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                              collate_fn=word_seq_collate_fn
+                              )
+
+    val_dataset = TrinityDataset_sentencelevel(args, args.val_data_path[0],
+                                 n_poses=args.n_poses,
+                                 subdivision_stride=args.subdivision_stride,
+                                 pose_resampling_fps=args.motion_resampling_framerate,
+                                 data_mean=args.data_mean, data_std=args.data_std)
+    test_loader = DataLoader(dataset=val_dataset, batch_size=args.batch_size,
+                             shuffle=False, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                             collate_fn=word_seq_collate_fn
+                             )
+    print("Data loaded successfully")
+    exit()
+    # build vocab
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    lang_model = build_vocab('words', [train_dataset, val_dataset], vocab_cache_path, args.wordembed_path,
+                             args.wordembed_dim)
+    train_dataset.set_lang_model(lang_model)
+    val_dataset.set_lang_model(lang_model)
+
+    # train
+    train_epochs(args, train_loader, test_loader, lang_model,
+                 pose_dim=82, trial_id=trial_id)
+
+
+if __name__ == '__main__':
+    _args = parse_args()
+    main({'args': _args})
diff --git a/scripts/train_text2embedding.py b/scripts/train_text2embedding.py
new file mode 100644
index 0000000..51d956d
--- /dev/null
+++ b/scripts/train_text2embedding.py
@@ -0,0 +1,332 @@
+import datetime
+import os
+import pprint
+import random
+import time
+import sys
+import numpy as np
+
+from torch.utils.data import DataLoader
+import torch
+[sys.path.append(i) for i in ['.', '..']]
+
+from model import vocab
+from model.seq2seq_net import Seq2SeqNet
+from model.text2embedding_model import text2embedding_model, text2embedding_model_New
+from train_eval.train_seq2seq import train_iter_seq2seq, train_iter_c2g_seq2seq, train_iter_text2embedding
+from utils.average_meter import AverageMeter
+from utils.vocab_utils import build_vocab
+
+from config.parse_args import parse_args
+
+from torch import optim
+
+from data_loader.lmdb_data_loader import *
+import utils.train_utils
+import torch.nn.functional as F
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+# device = 'cpu'
+import matplotlib.pyplot as plt
+
+
+debug = False
+
+
+
+
+def init_model(args, lang_model, pose_dim, _device):
+    n_frames = args.n_poses
+    try:
+        if args.text2_embedding_discrete == 'True':
+            pose_dim = int(args.autoencoder_vq_components)
+    except:
+        pass
+    generator = text2embedding_model(args, pose_dim, n_frames, lang_model.n_words, args.wordembed_dim,
+                           lang_model.word_embedding_weights).to(_device)
+    loss_fn = torch.nn.MSELoss()
+
+    return generator, loss_fn
+
+
+def train_epochs(args, train_data_loader, test_data_loader, lang_model, pose_dim, trial_id=None):
+    start = time.time()
+    loss_meters = [AverageMeter('loss'), AverageMeter('var_loss')]
+
+    # interval params
+    print_interval = np.max((int(len(train_data_loader) / 5),1))
+    save_model_epoch_interval = 10
+
+    # init model
+    generator, loss_fn = init_model(args, lang_model, pose_dim, device)
+    args_backup = args
+    start_epcoh = 1
+    Check_point_epoch = 0
+    load_pre_trained = False
+    if load_pre_trained:
+        Check_point_epoch = 250
+        addr = '{}/{}_checkpoint_{:03d}.bin'.format(args.model_save_path, args.name, Check_point_epoch)
+        txt2embedding_model = utils.train_utils.load_checkpoint_and_model(
+            addr, device, what='text2embedding')
+        args, generator, loss_fn, lang_model, pose_dim = txt2embedding_model
+        start_epcoh = Check_point_epoch
+    # define optimizers
+    gen_optimizer = optim.Adam(generator.parameters(), lr=args.learning_rate, betas=(0.5, 0.999))
+    args.epochs = args_backup.epochs
+
+
+    # training
+    global_iter = 0
+
+    # To record loss and evaluations metric:
+    val_metrics_list = []
+    loss_list = []
+    perplexity_list = []
+
+    for epoch in range(start_epcoh, Check_point_epoch + args.epochs+1):
+
+        # evaluate the test set
+        val_metrics = 0
+        val_metrics, perplexity = evaluate_testset(test_data_loader, generator, loss_fn, args)
+        val_metrics_list.append(val_metrics)
+        perplexity_list.append(perplexity)
+        # break
+        # save model
+        if epoch % save_model_epoch_interval == 0 and epoch > 0:
+            try:  # multi gpu
+                gen_state_dict = generator.module.state_dict()
+            except AttributeError:  # single gpu
+                gen_state_dict = generator.state_dict()
+
+            save_name = '{}/{}_checkpoint_{:03d}.bin'.format(args.model_save_path, args.name, epoch)
+
+            utils.train_utils.save_checkpoint({
+                'args': args, 'epoch': epoch, 'lang_model': lang_model,
+                'pose_dim': pose_dim, 'gen_dict': gen_state_dict
+            }, save_name)
+
+        # train iter
+        iter_start_time = time.time()
+
+        loss_epoch = AverageMeter('loss')
+        loss_epoch.reset()
+
+        for iter_idx, data in enumerate(train_data_loader, 0):
+            global_iter += 1
+
+            in_text, text_lengths, target_vec, in_audio, aux_info, sentence_leve_latents, c_portion, GPT3_Embedding = data
+            batch_size = target_vec.size(0)
+
+            # print('intext', in_text, '\nCindec', c_portion )
+
+            in_text = in_text.to(device)
+            in_audio = in_audio.to(device)
+            # target_vec = target_vec.to(device)
+            target_vec = sentence_leve_latents.to(device)
+            c_portion = c_portion.to(device)
+            GPT3_Embedding = GPT3_Embedding.to(device)
+            # train
+            loss = train_iter_text2embedding(args, epoch, in_text, text_lengths,
+                                             in_audio, target_vec, c_portion, GPT3_Embedding,
+                                             generator, gen_optimizer)
+
+            # loss values
+            loss_epoch.update(loss['loss'], batch_size)
+            for loss_meter in loss_meters:
+                name = loss_meter.name
+                if name in loss:
+                    loss_meter.update(loss[name], batch_size)
+
+            # print training status
+            if (iter_idx + 1) % print_interval == 0:
+                print_summary = 'EP {} ({:3d}) | {:>8s}, {:.0f} samples/s | '.format(
+                    epoch, iter_idx + 1, utils.train_utils.time_since(start),
+                    batch_size / (time.time() - iter_start_time))
+                for loss_meter in loss_meters:
+                    if loss_meter.count > 0:
+                        print_summary += '{}: {:.3f}, '.format(loss_meter.name, loss_meter.avg)
+                        loss_meter.reset()
+                logging.info(print_summary)
+
+            iter_start_time = time.time()
+        loss_list.append(loss_epoch.avg)
+
+    plot_loss(args, all_eval_loss=val_metrics_list,
+              all_train_loss=loss_list,
+              perplexities=(perplexity_list))
+
+def evaluate_testset(test_data_loader, generator, loss_fn, args):
+
+    # to evaluation mode
+    generator.train(False)
+
+    losses = AverageMeter('loss')
+    start = time.time()
+
+    perplexities = AverageMeter('perplexity')
+
+    with torch.no_grad():
+        for iter_idx, data in enumerate(test_data_loader, 0):
+            in_text, text_lengths, target_vec, in_audio, aux_info,\
+            sentence_leve_latents, cluster_ids, GPT3_embeddings = data
+
+            batch_size = target_vec.size(0)
+
+            if debug:
+                print("sentence_level_latents.shape", sentence_leve_latents.shape) # [128, 4, 400]
+                print("cluster_ids", cluster_ids.shape)
+            in_text = in_text.to(device)
+            in_audio = in_audio.to(device)
+            # target = target_vec.to(device)
+            target = sentence_leve_latents.to(device)
+            cluster_ids = cluster_ids.to(device)
+            GPT3_embeddings = GPT3_embeddings.to(device)
+            if generator.text2_embedding_discrete:
+                out_latents, _ = generator(in_text, text_lengths, in_audio,
+                                           cluster_ids, GPT3_embeddings, None)
+            else:
+                out_latents, _ = generator(in_text, text_lengths, in_audio,
+                                           target, None)
+
+            if args.text2_embedding_discrete == 'False':
+                if debug:
+                    print("!", out_latents.shape, out_latents)
+                loss = loss_fn(out_latents, target)
+                losses.update(loss.item(), batch_size)
+            else:
+
+                # 1. CrossEntropy Loss
+
+                cluster_targets_one_hot = F.one_hot(cluster_ids.reshape(-1).to(torch.int64), 514)
+
+                # cluster_targets_one_hot = cluster_targets_one_hot.reshape(os[0], os[1], -1)
+                if debug:
+                    print("check shape before reshape", out_latents.shape)
+                out_latents = out_latents.reshape(-1, out_latents.shape[2])
+
+                if debug:
+
+                    print("check shape", out_latents.shape,
+                          cluster_targets_one_hot.shape,
+                          cluster_ids.reshape(-1).shape)
+                # loss = torch.nn.MSELoss()(out_latents, cluster_targets_one_hot)
+                loss = torch.nn.CrossEntropyLoss()(out_latents, cluster_ids.reshape(-1).to(torch.long))
+                losses.update(loss.item(), batch_size)
+
+                # 2. Perplexity
+                encoding_indices = out_latents.argmax(1).float().unsqueeze(1)
+
+                encodings = torch.zeros(encoding_indices.shape[0], 514, device=encoding_indices.device)
+                encodings.scatter_(1, encoding_indices.to(torch.int64), 1)
+                avg_probs = torch.mean(encodings, dim=0)
+                perplexity = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-10)))
+                # perplexity = torch.nn.CrossEntropyLoss()(out_latents.float(),
+                #                                          cluster_ids.view(-1).long())
+                perplexity = perplexity.detach().cpu().numpy()
+                # print(out_latents.shape, avg_probs, perplexity)
+                perplexities.update(perplexity, batch_size)
+
+                # 3. BLEU Score
+                
+
+
+    # back to training mode
+    generator.train(True)
+
+    # print
+    elapsed_time = time.time() - start
+    logging.info('[VAL] loss: {:.3f} / {:.1f}s'.format(losses.avg, elapsed_time))
+
+    return losses.avg, perplexities.avg
+
+
+def plot_loss(args, all_eval_loss, all_train_loss, perplexities):
+
+
+    # X = np.arange(136-3)
+    # fig = plt.figure()
+    # ax = fig.add_axes([0, 0, 1, 1])
+    # ax.bar(X + 0.00, all_train_loss[0], color='b', width=0.25)
+    # ax.bar(X + 0.25, all_eval_loss[1], color='g', width=0.25)
+    # plt.show()
+    # plotting the second plot
+    plt.plot(all_train_loss, label='Train loss')
+    plt.plot(all_eval_loss, label='Evaluation loss')
+
+    # Labeling the X-axis
+    plt.xlabel('Epoch number')
+    # Labeling the Y-axis
+    plt.ylabel('Loss Average')
+    # Give a title to the graph
+    plt.title('Training/Evaluation Loss based on epoch number')
+
+    # Show a legend on the plot
+    plt.legend()
+
+    plt.savefig(os.path.join(args.model_save_path, 'loss_plot.png'))
+    plt.show()
+
+    plt.title("perplexities=" + str(perplexities[-1]))
+    plt.plot(perplexities, label='aa')
+    plt.savefig(os.path.join(args.model_save_path, 'Perplexity.png'))
+    plt.show()
+
+
+
+def main(config):
+    args = config['args']
+
+    args.model_save_path = os.path.dirname(args.autoencoder_checkpoint) + '/text2mbedding/'
+
+    trial_id = None
+
+    # random seed
+    if args.random_seed >= 0:
+        torch.manual_seed(args.random_seed)
+        torch.cuda.manual_seed_all(args.random_seed)
+        np.random.seed(args.random_seed)
+        random.seed(args.random_seed)
+        os.environ['PYTHONHASHSEED'] = str(args.random_seed)
+
+    # set logger
+    utils.train_utils.set_logger(args.model_save_path, os.path.basename(__file__).replace('.py', '.log'))
+
+    logging.info("PyTorch version: {}".format(torch.__version__))
+    logging.info("CUDA version: {}".format(torch.version.cuda))
+    logging.info(pprint.pformat(vars(args)))
+
+    # dataset
+    train_dataset = TrinityDataset_sentencelevel(args, args.train_data_path[0],
+                                   n_poses=args.n_poses,
+                                   subdivision_stride=args.subdivision_stride,
+                                   pose_resampling_fps=args.motion_resampling_framerate,
+                                   data_mean=args.data_mean, data_std=args.data_std)
+    train_loader = DataLoader(dataset=train_dataset, batch_size=args.batch_size,
+                              shuffle=True, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                              collate_fn=word_seq_collate_fn
+                              )
+
+    val_dataset = TrinityDataset_sentencelevel(args, args.val_data_path[0],
+                                 n_poses=args.n_poses,
+                                 subdivision_stride=args.subdivision_stride,
+                                 pose_resampling_fps=args.motion_resampling_framerate,
+                                 data_mean=args.data_mean, data_std=args.data_std)
+    test_loader = DataLoader(dataset=val_dataset, batch_size=args.batch_size,
+                             shuffle=False, drop_last=True, num_workers=args.loader_workers, pin_memory=True,
+                             collate_fn=word_seq_collate_fn
+                             )
+    print("Data loaded successfully")
+    # build vocab
+    vocab_cache_path = os.path.join(os.path.split(args.train_data_path[0])[0], 'vocab_cache.pkl')
+    lang_model = build_vocab('words', [train_dataset, val_dataset], vocab_cache_path, args.wordembed_path,
+                             args.wordembed_dim)
+    train_dataset.set_lang_model(lang_model)
+    val_dataset.set_lang_model(lang_model)
+
+    # train
+    train_epochs(args, train_loader, test_loader, lang_model,
+                 pose_dim=15*9, trial_id=trial_id)
+
+
+if __name__ == '__main__':
+    _args = parse_args()
+    main({'args': _args})
diff --git a/scripts/trinity_data_to_lmdb.py b/scripts/trinity_data_to_lmdb.py
new file mode 100644
index 0000000..6baf273
--- /dev/null
+++ b/scripts/trinity_data_to_lmdb.py
@@ -0,0 +1,161 @@
+import argparse
+import os
+import glob
+from pathlib import Path
+
+import librosa
+import numpy as np
+import lmdb
+import pyarrow
+from sklearn.pipeline import Pipeline
+
+from pymo.parsers import BVHParser
+from pymo.preprocessing import *
+from pymo.viz_tools import *
+from pymo.writers import *
+
+from scipy.spatial.transform import Rotation as R
+
+import joblib as jl
+
+from utils.data_utils import SubtitleWrapper, normalize_string
+
+target_joints = ['Spine', 'Spine1', 'Spine2', 'Spine3', 'Neck', 'Neck1', 'Head',
+                 'RightShoulder', 'RightArm', 'RightForeArm', 'RightHand',
+                 'LeftShoulder', 'LeftArm', 'LeftForeArm', 'LeftHand']
+rep_learning = False
+
+DATASET_Type = 'Trinity'
+DATASET_Type = 'TWH'
+
+def process_bvh(gesture_filename):
+    p = BVHParser()
+
+    data_all = list()
+    data_all.append(p.parse(gesture_filename))
+
+    data_pipe = Pipeline([
+        ('dwnsampl', DownSampler(tgt_fps=20, keep_all=False)),
+        ('root', RootTransformer('hip_centric')),
+        ('mir', Mirror(axis='X', append=True)),
+        ('jtsel', JointSelector(target_joints, include_root=True)),
+        ('cnst', ConstantsRemover()),
+        # ('np', Numpyfier())
+    ])
+
+    out_data = data_pipe.fit_transform(data_all)
+    jl.dump(data_pipe, os.path.join('../resource', 'data_pipe.sav'))
+
+    # euler -> rotation matrix
+    out_data = out_data.reshape((out_data.shape[0], out_data.shape[1], -1, 3))
+    out_matrix = np.zeros((out_data.shape[0], out_data.shape[1], out_data.shape[2], 9))
+    for i in range(out_data.shape[0]):  # mirror
+        for j in range(out_data.shape[1]):  # frames
+            r = R.from_euler('ZXY', out_data[i, j], degrees=True)
+            out_matrix[i, j] = r.as_matrix().reshape(out_data.shape[2], 9)
+    out_matrix = out_matrix.reshape((out_data.shape[0], out_data.shape[1], -1))
+
+    return out_matrix[0], out_matrix[1]
+
+
+def make_lmdb_gesture_dataset(base_path):
+    gesture_path = os.path.join(base_path, 'Motion')
+    audio_path = os.path.join(base_path, 'Audio')
+    text_path = os.path.join(base_path, 'Transcripts')
+    if not rep_learning:
+        out_path = os.path.join(base_path, 'lmdb')
+    else:
+        out_path = os.path.join(base_path, 'lmdb_rep_learning')
+    if not os.path.exists(out_path):
+        os.makedirs(out_path)
+
+    map_size = 1024 * 20  # in MB
+    map_size <<= 20  # in B
+    db = [lmdb.open(os.path.join(out_path, 'lmdb_train'), map_size=map_size),
+          lmdb.open(os.path.join(out_path, 'lmdb_test'), map_size=map_size)]
+
+    # delete existing files
+    for i in range(2):
+        with db[i].begin(write=True) as txn:
+            txn.drop(db[i].open_db())
+
+    all_poses = []
+    bvh_files = sorted(glob.glob(gesture_path + "/*.bvh"))
+    for v_i, bvh_file in enumerate(bvh_files):
+        name = os.path.split(bvh_file)[1][:-4]
+        print(name)
+
+        # load skeletons and subtitles
+        poses, poses_mirror = process_bvh(bvh_file)
+        subtitle = SubtitleWrapper(os.path.join(text_path, name + '.json')).get()
+
+        # load audio
+        audio_raw, audio_sr = librosa.load(os.path.join(audio_path, '{}.wav'.format(name)),
+                                           mono=True, sr=16000, res_type='kaiser_fast')
+
+        # process
+        clips = [{'vid': name, 'clips': []},  # train
+                 {'vid': name, 'clips': []}]  # validation
+
+        # split
+        if v_i == 0:
+            dataset_idx = 1  # validation
+        else:
+            dataset_idx = 0  # train
+
+        # word preprocessing
+        word_list = []
+        for wi in range(len(subtitle)):
+            word_s = float(subtitle[wi]['start_time'][:-1])
+            word_e = float(subtitle[wi]['end_time'][:-1])
+            word = subtitle[wi]['word']
+
+            word = normalize_string(word)
+            if len(word) > 0:
+                word_list.append([word, word_s, word_e])
+
+        # save subtitles and skeletons
+        poses = np.asarray(poses, dtype=np.float16)
+        clips[dataset_idx]['clips'].append(
+            {'words': word_list,
+             'poses': poses,
+             'audio_raw': audio_raw
+             })
+        poses_mirror = np.asarray(poses_mirror, dtype=np.float16)
+        clips[dataset_idx]['clips'].append(
+            {'words': word_list,
+             'poses': poses_mirror,
+             'audio_raw': audio_raw
+             })
+
+        # write to db
+        for i in range(2):
+            with db[i].begin(write=True) as txn:
+                if len(clips[i]['clips']) > 0:
+                    k = '{:010}'.format(v_i).encode('ascii')
+                    v = pyarrow.serialize(clips[i]).to_buffer()
+                    txn.put(k, v)
+
+        all_poses.append(poses)
+
+    # close db
+    for i in range(2):
+        db[i].sync()
+        db[i].close()
+
+    # calculate data mean
+    all_poses = np.vstack(all_poses)
+    pose_mean = np.mean(all_poses, axis=0)
+    pose_std = np.std(all_poses, axis=0)
+
+    print('data mean/std')
+    print(str(["{:0.5f}".format(e) for e in pose_mean]).replace("'", ""))
+    print(str(["{:0.5f}".format(e) for e in pose_std]).replace("'", ""))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("db_path", type=Path)
+    args = parser.parse_args()
+
+    make_lmdb_gesture_dataset(args.db_path)
diff --git a/scripts/twh_dataset_to_lmdb.py b/scripts/twh_dataset_to_lmdb.py
new file mode 100644
index 0000000..9e83fae
--- /dev/null
+++ b/scripts/twh_dataset_to_lmdb.py
@@ -0,0 +1,279 @@
+import argparse
+import glob
+import os
+from pathlib import Path
+
+import librosa
+import lmdb
+import pyarrow
+from sklearn.pipeline import Pipeline
+import joblib as jl
+
+from pymo.preprocessing import *
+from pymo.parsers import BVHParser
+from utils.data_utils import SubtitleWrapper, normalize_string
+
+# 18 joints (only upper body)
+target_joints = ['b_spine0', 'b_spine1', 'b_spine2', 'b_spine3', 'b_l_shoulder', 'b_l_arm', 'b_l_arm_twist', 'b_l_forearm', 'b_l_wrist_twist', 'b_l_wrist', 'b_r_shoulder', 'b_r_arm', 'b_r_arm_twist', 'b_r_forearm', 'b_r_wrist_twist', 'b_r_wrist', 'b_neck0', 'b_head']
+
+# 24 joints (upper and lower body excluding fingers)
+# target_joints = ['body_world', 'b_root', 'b_l_upleg', 'b_l_leg', 'b_r_upleg', 'b_r_leg', 'b_spine0', 'b_spine1', 'b_spine2', 'b_spine3', 'b_l_shoulder', 'b_l_arm', 'b_l_arm_twist', 'b_l_forearm', 'b_l_wrist_twist', 'b_l_wrist', 'b_r_shoulder', 'b_r_arm', 'b_r_arm_twist', 'b_r_forearm', 'b_r_wrist_twist', 'b_r_wrist', 'b_neck0', 'b_head']
+
+# 56 joints (upper and lower body including fingers)
+# target_joints = ['body_world', 'b_root', 'b_l_upleg', 'b_l_leg', 'b_r_upleg', 'b_r_leg', 'b_spine0', 'b_spine1', 'b_spine2', 'b_spine3', 'b_l_shoulder', 'b_l_arm', 'b_l_arm_twist', 'b_l_forearm', 'b_l_wrist_twist', 'b_l_wrist', 'b_l_pinky1', 'b_l_pinky2', 'b_l_pinky3', 'b_l_ring1', 'b_l_ring2', 'b_l_ring3', 'b_l_middle1', 'b_l_middle2', 'b_l_middle3', 'b_l_index1', 'b_l_index2', 'b_l_index3', 'b_l_thumb0', 'b_l_thumb1', 'b_l_thumb2', 'b_l_thumb3', 'b_r_shoulder', 'b_r_arm', 'b_r_arm_twist', 'b_r_forearm', 'b_r_wrist_twist', 'b_r_wrist', 'b_r_thumb0', 'b_r_thumb1', 'b_r_thumb2', 'b_r_thumb3', 'b_r_pinky1', 'b_r_pinky2', 'b_r_pinky3', 'b_r_middle1', 'b_r_middle2', 'b_r_middle3', 'b_r_ring1', 'b_r_ring2', 'b_r_ring3', 'b_r_index1', 'b_r_index2', 'b_r_index3', 'b_neck0', 'b_head']
+
+
+def process_bvh(gesture_filename, dump_pipeline=False):
+    p = BVHParser()
+
+    data_all = list()
+    data_all.append(p.parse(gesture_filename))
+
+    data_pipe = Pipeline([
+        ('dwnsampl', DownSampler(tgt_fps=30, keep_all=False)),
+        ('root', RootNormalizer()),
+        ('jtsel', JointSelector(target_joints, include_root=False)),
+        # ('mir', Mirror(axis='X', append=True)),
+        ('np', Numpyfier())
+    ])
+
+    out_data = data_pipe.fit_transform(data_all)
+    if dump_pipeline:
+        jl.dump(data_pipe, os.path.join('../resource', 'data_pipe.sav'))
+
+    # euler -> rotation matrix
+    out_data = out_data.reshape((out_data.shape[0], out_data.shape[1], -1, 6))  # 3 pos (XYZ), 3 rot (ZXY)
+    out_matrix = np.zeros((out_data.shape[0], out_data.shape[1], out_data.shape[2], 12))  # 3 pos, 1 rot matrix (9 elements)
+    for i in range(out_data.shape[0]):  # mirror
+        for j in range(out_data.shape[1]):  # frames
+            for k in range(out_data.shape[2]):  # joints
+                out_matrix[i, j, k, :3] = out_data[i, j, k, :3]  # positions
+                r = R.from_euler('ZXY', out_data[i, j, k, 3:], degrees=True)
+                out_matrix[i, j, k, 3:] = r.as_matrix().flatten()  # rotations
+    out_matrix = out_matrix.reshape((out_data.shape[0], out_data.shape[1], -1))
+
+    return out_matrix[0]
+
+def process_bvh_rot_only(gesture_filename, dump_pipeline=False):
+    p = BVHParser()
+
+    data_all = list()
+    data_all.append(p.parse(gesture_filename))
+
+    data_pipe = Pipeline([
+        ('dwnsampl', DownSampler(tgt_fps=30, keep_all=False)),
+        ('root', RootNormalizer()),
+        ('jtsel', JointSelector(target_joints, include_root=False)),
+        # ('mir', Mirror(axis='X', append=True)),
+        ('cnst', ConstantsRemover()),
+        ('exp', MocapParameterizer('expmap')),
+        ('np', Numpyfier())
+    ])
+
+    out_data = data_pipe.fit_transform(data_all)
+    jl.dump(data_pipe, os.path.join('../resource', 'data_pipe.sav'))
+
+    # euler -> rotation matrix
+    out_data = out_data.reshape((out_data.shape[0], out_data.shape[1], -1, 3))
+    out_matrix = np.zeros((out_data.shape[0], out_data.shape[1], out_data.shape[2], 9))
+    for i in range(out_data.shape[0]):  # mirror
+        for j in range(out_data.shape[1]):  # frames
+            r = R.from_euler('ZXY', out_data[i, j], degrees=True)
+            out_matrix[i, j] = r.as_matrix().reshape(out_data.shape[2], 9)
+    out_matrix = out_matrix.reshape((out_data.shape[0], out_data.shape[1], -1))
+
+    return out_matrix[0]
+
+
+def process_bvh_rot_only_Taras(gesture_filename, dump_pipeline=False):
+    p = BVHParser()
+
+    data_all = list()
+    data_all.append(p.parse(gesture_filename))
+
+    data_pipe = Pipeline([
+        ('dwnsampl', DownSampler(tgt_fps=10, keep_all=False)),
+        ('root', RootNormalizer()),
+        ('jtsel', JointSelector(target_joints, include_root=False)),
+        # ('mir', Mirror(axis='X', append=True)),
+        # ('cnst', ConstantsRemover()),
+        ('exp', MocapParameterizer('expmap')),
+        ('np', Numpyfier())
+    ])
+
+    out_data = data_pipe.fit_transform(data_all)
+    jl.dump(data_pipe, os.path.join('../resource', 'data_pipe.sav'))
+    return out_data[0]
+
+    # euler -> rotation matrix
+    out_data = out_data.reshape((out_data.shape[0], out_data.shape[1], -1, 3))
+    out_matrix = np.zeros((out_data.shape[0], out_data.shape[1], out_data.shape[2], 9))
+    for i in range(out_data.shape[0]):  # mirror
+        for j in range(out_data.shape[1]):  # frames
+            r = R.from_euler('ZXY', out_data[i, j], degrees=True)
+            out_matrix[i, j] = r.as_matrix().reshape(out_data.shape[2], 9)
+    out_matrix = out_matrix.reshape((out_data.shape[0], out_data.shape[1], -1))
+
+    return out_matrix[0]
+
+
+def process_bvh_test1(gesture_filename, dump_pipeline=False):
+    p = BVHParser()
+
+    data_all = list()
+    data_all.append(p.parse(gesture_filename))
+
+    data_pipe = Pipeline([
+        ('dwnsampl', DownSampler(tgt_fps=10, keep_all=False)),
+        ('root', RootNormalizer()),
+        ('jtsel', JointSelector(target_joints, include_root=False)),
+        # ('mir', Mirror(axis='X', append=True)),
+        ('cnst', ConstantsRemover()),
+        ('exp', MocapParameterizer('expmap')),
+        ('np', Numpyfier())
+    ])
+
+    out_data = data_pipe.fit_transform(data_all)
+    jl.dump(data_pipe, os.path.join('../resource', 'data_pipe.sav'))
+
+
+    # euler -> rotation matrix
+    out_data = out_data.reshape((out_data.shape[0], out_data.shape[1], -1, 3))
+    out_matrix = np.zeros((out_data.shape[0], out_data.shape[1], out_data.shape[2], 9))
+    for i in range(out_data.shape[0]):  # mirror
+        for j in range(out_data.shape[1]):  # frames
+            r = R.from_euler('ZXY', out_data[i, j], degrees=True)
+            out_matrix[i, j] = r.as_matrix().reshape(out_data.shape[2], 9)
+    out_matrix = out_matrix.reshape((out_data.shape[0], out_data.shape[1], -1))
+
+    return out_matrix[0]
+
+def make_lmdb_gesture_dataset(base_path):
+    gesture_path = os.path.join(base_path, 'bvh')
+    audio_path = os.path.join(base_path, 'wav')
+    text_path = os.path.join(base_path, 'tsv')
+    out_path = os.path.join(base_path, 'lmdb')
+    if not os.path.exists(out_path):
+        os.makedirs(out_path)
+
+    map_size = 1024 * 20  # in MB
+    map_size <<= 20  # in B
+    db = [lmdb.open(os.path.join(out_path, 'lmdb_train'), map_size=map_size),
+          lmdb.open(os.path.join(out_path, 'lmdb_test'), map_size=map_size)]
+
+    # delete existing files
+    for i in range(2):
+        with db[i].begin(write=True) as txn:
+            txn.drop(db[i].open_db())
+
+    all_poses = []
+    bvh_files = sorted(glob.glob(gesture_path + "/*.bvh"))
+    save_idx = 0
+    counter =0
+    for bvh_file in bvh_files:
+
+        # Payam: for check
+        if counter > 50:
+            break
+        counter = counter + 1
+
+
+        name = os.path.split(bvh_file)[1][:-4]
+        # name = name[:-3] + '114'
+        print(name)
+
+        # load subtitles
+        tsv_path = os.path.join(text_path, name + '.tsv')
+        if os.path.isfile(tsv_path):
+            subtitle = SubtitleWrapper(tsv_path).get()
+        else:
+            continue
+
+        # load audio
+        wav_path = os.path.join(audio_path, '{}.wav'.format(name))
+        if os.path.isfile(wav_path):
+            audio_raw, audio_sr = librosa.load(wav_path, mono=True, sr=16000, res_type='kaiser_fast')
+        else:
+            continue
+
+        # load skeletons
+        dump_pipeline = (save_idx == 0)
+        # poses, poses_mirror = process_bvh(bvh_file)
+        poses = process_bvh_test1(bvh_file, dump_pipeline)
+
+        # process
+        clips = [{'vid': name, 'clips': []},  # train
+                 {'vid': name, 'clips': []}]  # validation
+
+        # split
+        if save_idx % 100 == 0:
+            dataset_idx = 1  # validation
+        else:
+            dataset_idx = 0  # train
+
+        # word preprocessing
+        word_list = []
+        for wi in range(len(subtitle)):
+            try:
+                word_s = float(subtitle[wi][0])
+                word_e = float(subtitle[wi][1])
+                word = subtitle[wi][2].strip()
+            except:
+                print("*****************************Error in readong file line: ", wi)
+                continue
+
+            word_tokens = word.split()
+
+            for t_i, token in enumerate(word_tokens):
+                token = normalize_string(token)
+                if len(token) > 0:
+                    new_s_time = word_s + (word_e - word_s) * t_i / len(word_tokens)
+                    new_e_time = word_s + (word_e - word_s) * (t_i + 1) / len(word_tokens)
+                    word_list.append([token, new_s_time, new_e_time])
+
+        # save subtitles and skeletons
+        poses = np.asarray(poses, dtype=np.float16)
+        clips[dataset_idx]['clips'].append(
+            {'words': word_list,
+             'poses': poses,
+             'audio_raw': audio_raw
+             })
+        # poses_mirror = np.asarray(poses_mirror, dtype=np.float16)
+        # clips[dataset_idx]['clips'].append(
+        #     {'words': word_list,
+        #      'poses': poses_mirror,
+        #      'audio_raw': audio_raw
+        #      })
+
+        # write to db
+        for i in range(2):
+            with db[i].begin(write=True) as txn:
+                if len(clips[i]['clips']) > 0:
+                    k = '{:010}'.format(save_idx).encode('ascii')
+                    v = pyarrow.serialize(clips[i]).to_buffer()
+                    txn.put(k, v)
+
+        all_poses.append(poses)
+        save_idx += 1
+
+    # close db
+    for i in range(2):
+        db[i].sync()
+        db[i].close()
+
+    # calculate data mean
+    all_poses = np.vstack(all_poses)
+    pose_mean = np.mean(all_poses, axis=0, dtype=np.float64)
+    pose_std = np.std(all_poses, axis=0, dtype=np.float64)
+
+    print('data mean/std')
+    print('data_mean:', str(["{:0.5f}".format(e) for e in pose_mean]).replace("'", ""))
+    print('data_std:', str(["{:0.5f}".format(e) for e in pose_std]).replace("'", ""))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("db_path", type=Path)
+    args = parser.parse_args()
+
+    make_lmdb_gesture_dataset(args.db_path)
diff --git a/scripts/utils/Unityfier.py b/scripts/utils/Unityfier.py
new file mode 100644
index 0000000..9eec77a
--- /dev/null
+++ b/scripts/utils/Unityfier.py
@@ -0,0 +1,37 @@
+# This file converts transcript to a txt file that can be read by the unity
+import json
+import glob
+import os
+from data_utils import SubtitleWrapper, normalize_string
+
+jsons_path = '/local-scratch/pjomeyaz/GENEA_DATASET/trinityspeechgesture.scss.tcd.ie/data/GENEA_Challenge_2020_data_release/Test_data/Transcripts'
+
+json_output_path = jsons_path + '/Unity'
+if not os.path.exists(json_output_path):
+    os.makedirs(json_output_path)
+
+
+
+json_files = sorted(glob.glob(jsons_path + "/*.json"))
+
+
+for jfile in json_files:
+    name = os.path.split(jfile)[1][:-5]
+    print(name)
+
+    subtitle = SubtitleWrapper(jfile).get()
+    str_subtitle = ''
+    for word_boundle in subtitle:
+        start_time = word_boundle['start_time'][:-1] #Removing 's'
+        end_time = word_boundle['end_time'][:-1]     #Removing 's'
+        word = word_boundle['word']
+        str_subtitle += "{},{},{}\n".format(start_time, end_time, word)
+
+    str_subtitle = str_subtitle[:-1]
+
+    file2write = open(json_output_path + '/' + name + '.txt', 'w')
+    file2write.write(str_subtitle)
+    file2write.flush()
+    file2write.close()
+
+    print()
\ No newline at end of file
diff --git a/scripts/utils/__pycache__/average_meter.cpython-37.pyc b/scripts/utils/__pycache__/average_meter.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..06e06909f3c6a5b5147de75f998883846a13a34e
GIT binary patch
literal 1207
zcmZuxJ8u&~5Z=8zpB?8VLZU>1!WH6*a6&>NLI?$1q{2-^<7T<su3y~4?CwR1V>clU
z4Zi_O{?fHnsp(Sj&DpWyh_z;RW_EY>n{U?lVRN&|FuosMeg5Py_J^Fi;lVhDm|dt7
zCYj{d5XHW^T`rwd7Pv81mcxa03^A{u6vH0YbtLb)a#Om}yXIX_)}(*Ux;42a8*&X1
zUpD1BB6YceeB0&5`*WqEzWS)F(vz3%JRKKSnO2m^ma(~pX$Ptm&39GdxX@Z<wsjsQ
zg}Qa#^kM=+gJu-P+7NR9HRV$#*wlfB_0##qILgAsyc4*kmqn>6-Ck<T8ZJ_nVn!uB
zO^Gzo@tf&7rrWD9$>S(FG_j6sJP605JXIIbB-FVXD$!TQ7FvljmBoWJ(qF<j$ulLx
zc2XD=Hq82A3M*{qho@tu;(<6>963=L2(ZIE3z2*{wk9m>FfrkxWRVgxFUA*TLx^EE
zv_cRt<3lm-<2Tv6&*6)^T`{K}7&I?gc?w5XtW=s;8chR3ZpDn#xuHXWSGpj$bffcr
z>6#)fYjIv=HsG3Wu2`9?2Dvq;HHaZ^4hz6`R)AF$-w>LWhEX+D!;WAq8pb>};;@}v
z_8Qo!d5Abur}EHFtipf7max%KPp>0`-Xyg}Y6B{8=XhzU<k<t<rC*F?WYsoSm(X|k
z7N0y`fj-~0WR%#XCsM(F27~w;v3(|;eL$)#cc0Br!ie`M;{4!vbzWz(eo2pR*180_
zllrXn{1OCDoLK;%#6;=$a-Bw&x-Ew>x*a%G1zjKS)L8wng1Qih3bBX7hB7dIz$d#a
z@E2P;XdXewL<I*8x`f~eQ8tB0b6F&0ZwT?Vh?2z$E|7UFgr@#hjn<FIqL0KqF!VRk
t)mylGZ-%}!4es$~t>HI)P4Q0Plzx6TQn9W4|0O6JZ|6|n5!k;F_7CVy4$}Yt

literal 0
HcmV?d00001

diff --git a/scripts/utils/__pycache__/data_utils.cpython-37.pyc b/scripts/utils/__pycache__/data_utils.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d20505d55060ef1d9671938aab8ab8033a931817
GIT binary patch
literal 2073
zcmaJ>&2Jnv6u0MVC!0;1l&Caq4?|U{9h%KTD<P<&lv0`siLMHXl(I<}CS!Lq>3r4p
zCh0IMB(@j0aOad9`Img<LeAWJ;@Pta=?XC;zYmY?_vZJ0eqYSYG!Z<%eERbbzdH#1
zshrct1mklU{6i1~5uBi2t#J<%gP3nn&v=4}MeH|-*dwE761;|d=LFWHZiJPBM`;_z
zBN+T^5EVK?$LI(jqahN;5$qyHg^ta!XA<k2XA|d~*QmXEZ-pS5Ti)}M{D873<aE&!
zEKV1_Fe4tLX}(XrEYI2r6@oG^+6`G4fyKSjy}h}(^udE|zqGuqkG;~qy?JZfm#CcE
z{61`#58J(u+Mnqaciv*JG`IL2-;%bDByA313TZGZO+MK1vBdlj#HQfi-u~1{@+eH&
zJYu1Uc00vhp3>K0*<m@4Y0#%!3>Xb|Xcq0JA$!@0l02iN^C%f`NG;C#ofNFjqrAOV
zP#Wz9PsUrG(2T;Fah`SH@VF4XGZ1mYJ0uigpv|S?wVcgzmWD}O(m<t|^%eHK1p*l@
zd_ljLFq;NHQj@38S-C%f8H%99P+n>tSp*;Bo;@zOQ<u93g&++efJ<C51$v4!p%~Nh
z;@V(G#3G>^EG!Dj${V-0$Q}Q|{Rg0bygT&okM-u#;<hsH4gJU%=b*GoLo;O_j6QHB
zP<+7^B!Pi_5CnI9L%M*wjw9!jJS2c?20??<Sy*9ln!J!Rr!zqjid|izS}5L2HwfY^
z7C~?cJZ^zN*umwMiJw?HUG@&A6|Z%saB(2Vy>V`s&4BM?b`jKMK(kN9WtivsmvQ;t
zWF+{021f%HDo*Jjzk>04PoN5q@NdtDhQLQgWgO$7dAM4cU!v`mAylfeh_Qf(xqx0e
z?E1eR!3$_;A6}_!SUEd{E42retsDjFtJMwk>iP!S0C>Ma-=gPWWmhdE?3YdU2E+YT
zz(XO-?@(quuZv)HrG1by!euK=fVUYCbDwgiK=B=E<ps^8wa0Uisz4eP=_pQQM9!_o
z3GL=$H6LW;F=ILN8#P|DuwPDTM!$eH(o&|hVnNehi-iYg6_%C?KY=g10Nby^;EFv6
z8@Q?8Sv(8+25`T;If?jx(dVoLE}U}KyY|5mii>E^06rQ+C;;~a^sf=XD6A0xC2Zl;
zSEVwJ8pmj4jG#mn^t1&8($><^vax_(SX<W68rnl=*nncW^Qf}P)I540RSt{>X<h?c
zU94_-<$&c`-xG04Jubqu@PN?@`FBXEc(n$DD<CVxTEG}Ig0`$%v(c>!grym0LiMe~
z2?V9PoP_BP30K;1z~3-X*`9<9NE&b&<r(2i%evk16)@_3I>L$K!HG)dL|y2377BB+
ztgOGl_g$v$jI>kLBIVrrS5F_Wtv`JFEO_>Cef{ybz^wv0myi}cj6o(b7t&0_LOOkx
z4+;(ksN2<>RvZWvBmwO;wcZbt0hNwcT-v+&fN|MK<7^-(7~`~MD~%I4w*HAp45rWh
z;4O&56|E4k$HFdljLZ0@QC|BWP`Z9owU#V|yaL$_f;1-s=uUxb1;ML9nABft5E@aQ
z7C?2TBC$CoR5#L8W_OjPdcZlMIu}#@TtUE<=i;W}TJX;3dEa8H$LQmApVzbJPS6L|
T4RE<mZAI0KtD?BrHQnYv5(NQ~

literal 0
HcmV?d00001

diff --git a/scripts/utils/__pycache__/data_utils_twh.cpython-37.pyc b/scripts/utils/__pycache__/data_utils_twh.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2706833d5c0fae2b722b713a5411e1791083159d
GIT binary patch
literal 2227
zcma)7Pj4GV6rY*hwbxG4CZ&RgipoVu#59gkix5;%+7j9dskI7;l;Sk3HapXJ)BV?(
zandg9lgkZ>uaI1jxbhkJ3>@alg`Bzd0`Ki6B{fpTTJzqUzi;OKe((8<nVBYm?~lKK
z+y31Jjh&Ov0^=q$brl3BoJOQyY1*e{pY<7M+<HO!)?LDF?z|w}8MD64=^AO*j$j|@
z1r!y0s2yml(9}H;1vwyx<bWQM5z*|x;*_(399qXc>$M)7erm2(^{TxE0dMvES}RKT
zg$#Tp7F$||@nXwQcuR^n-4m@OO*)a#TF6!~@TDJsrCPbWu(`N&`Nme;klb0={BWzC
z&uxC^ck)}E{>Poq%egBrX)CuLsVi;UIHeEcC~%M&CWW!oaJx+nRlg9~fLFa7ccV1$
zqmBxsuY*B1+f8Hf+|Ro*RiW^9gwjJPyls&LgV>i(x<QmCf_GP=p@K_=$xb&0tGk+Z
z)-oZ2fp_<K%UzKOI5SL>E*u_aT6Kpyj8vEV+V@J+)BA&^?75juQW^VEm<tcDn(Wjt
z>eC<yn`UPy{?1W31^oaAPrg^6e+nalAkcj#tOJ1bkoKKpxN8;I{W`DlI>7I8mp4E+
zcoU$QGH2I@+d9;d*pPmf37NmY@Q7b&-?)AQ^!Has?d!*Sb7^r4&1<7}z>e>L8n03E
z<UANVz!VSJLJPPA1rCA}+G{i8LhMUFW-dy7?rF8>oeaUWtwi+L%$$sPnXd;WMwLM7
z8`tx~B-EaVo|FxO&>GDzOuS&}WZ7$UMoO30#OM}e_xNH|&OpCS<ylaZ4y_%bFTl7e
z-%IniCOyIYt5{&ZAgM|xzk>1nM@<Skra$E)_DmM+OR}{)vI?ur1k?<Hk~a%WLq3fG
zz8}~;8ClxqYz(o36%Mx!>B!brVZ*%rgvqS14<Ms3wy>^N*t|YZ-XZ0BvRglZ5uAH^
zZi8$<^uH$GkcVI;@7hGWPnz;)3dxPQ08aI9Nx~k&IpcB!kaBx@BS;f165VLzAHDIr
zEoCZOtJUh4v~h$%GE5Be&i~kVFIBp=kxEE<M5f&^cAAMqVie7^=ix2<Fj1aotVkrr
z$sjW{By$#p+SHJ$2`{lTb9cf>^iq8%9VWcIRokiHoW;ZJ$1rW|C`<&95Vvc=Cy<tB
zVC6+<>JkV-8Ex84>e42iE#)*??2>ziPSgDIBn$q-=U1G7I{j}Q-Gh$C%pkiAvV)CS
z!Bnfl9??V;w4-ZP;uh?nen`f^1mq02=YbF0DP^scfcs<nkv+0U&Zsu30~6fyq;PoS
zJn*5YL96rTMX)i=d-;WZnI=0e9mb-iv>#_JNHM5k$!l4H7_LE67`lSUX`nEWYYqNV
z;274cz@o+iFtCz7n+T+Fmm@#k=KjhG9Pk%3l6NNK1zF=M0SI%ov|JWowE{-9uk>()
zoRFC2M_69MS-|IL8LdB|-*zP?v2kK7+GcM3tNXXt)^FW^;61ptzJB|go}9;NTRa1*
z0Psp1EA}%}+mY!oQ?MPA-AwHR{X7haCrq={dww(&rdBFtoIyI2%GBd98EOH>@ukgl
z97b?#^@Pa?Or3h(XW&O6w+Pg58>m!cZ_^K0e(?<<>9w0!=@K9A<(a1E#VH>~XhT{&
z9r{r<Q-z=&q;UrPmJ8rR&Y?Jm0)5I)P+&<qC9sf6{BOav3VXnE=Gw01zE_U79b*M*
l&)B;vO-d)_KTkP#>YVb}swyv67hFNi6f)k0hr)SN{{WXiC20Tv

literal 0
HcmV?d00001

diff --git a/scripts/utils/__pycache__/train_utils.cpython-37.pyc b/scripts/utils/__pycache__/train_utils.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c7d5058b79b9c7ba8edd13e0c27c368c7e18b898
GIT binary patch
literal 2772
zcmai0-EJGl72eq&a!FAXP5DoD8mMiQhDB|*?oDB&X=K@PQv?DUHwmz4u~_d6sg?GJ
znHkEmN&-cl_M)hPpbt<`mwksmK+#8-+X4l2->Y8bJF}D}yFj(X%$%9|=FFKhXTHM+
ztE+7X-_377Oa9bxoPSa0?DHUeh@$?1iaVU;PE@xnVz%!_E?PJD3P18$Jr_g`JJ*bw
zwhg1uwymgz*2~+)O4PAfFJFb+UCL{adrSEW<o;4#hkSJ@UxR#oDc^v66Y>=v@Wxjx
z+Tcwde&s~Bc#F5u-sUU3gZ3@n=WDzN#J4zm;tbYLaT;fLz|8uml}>b4jz7wB@$;nQ
zxsVucXJw}2qT(VqUS5q;mlojjQJN?_DC*Csj^5GEKHDcBIDc?AoQKYDdUKb?7T(M~
zVa)lm|AnKSg@59V+1#4}chHht2m1}kjJq#g=FGfBQ->I9E!s0@#!g)Bai0e-J<<Se
zD<~b_n0dVU(xZ{ZDi2TCqKol0!f*%YYVH0C#)sl4Q#!5=GRIjsrva=SxKUUnM-QHB
zp`w+iNqR7y>{QcIAK%=1FHxz^3NcXkH@Ds^gi^^Eoq=b9TpYpyjZr0wM4NE0_Kz}c
zbSHVG^X%!MXPR)(7%nuvtST*A*w3s=QB;R#9yO~^v=m9Pbg*$NWnj;N;F(mWkxV9{
z<dS@B{7ItsjK5!HrGcR;OLwYrl#NaB@X>?M9-G!!=ufbpkS0_Rt%c1mwm;qdc=z$Z
zk3tv|la8oE(g@dDCWSD~rSaChzd7<2Hv8}IhIy4H`5l$YM5lYh$$nLc=gIL<Rw@(m
zSSUS}B7Q2$bgxL{!7$CMQt;tUK2=&soOxJ48tzngo=k*D_u|LPC67fZpfjt=A(Uqm
zt%g&b<!Y#9l9jQQ?@peZHqH=}|Ag$rB1M(MuCpF%vygR}&u+0klW$=(Wi*G6{6ukR
zWAI%Fhz3TnblW*$b4WAB*;VH!&K&P@hW^OCPSN4*p1u$CadV3on_FBBT;t;+_e?O#
zt4gBFlwpN+O<mmv+_V!F7g;$)=)8@-YM^ppT8GK+Kzd~sQ3Vxi78wk)oD&BI!GxR6
z+~rKOnfroSW8B?icgvbUv1c_=#Hp-Ih4ISjncRem|FMO@u?4+@>7^|pTqVg7ub$~^
zE~2ZhT|^HHkH=kjoV|K9u;l8K<WTTUy(c!)J&_(v5Pf>!n?P4GMKmf*%N3|GepZeu
zcq&k!4PnBw_<GaOHChs%EBiiKOPA)5-+n-<ymT@(PD(DLk2si~s&(IAan~9*O80ug
zUJwVk```}TEcgN};&E@onfqvcv;o=xE%8GGZ4+yoSR2%9XD$y9TzRPdnLlgHnltvo
z|Ft_0=dD?|ptY?H?82O{S=j@&4eUZU{y(<-H;^kp{!=$ELH-@c4v_!R;U&nw0J(bL
ze$C{jZe4;M0o%3MPjved?0sO@EcQsRT!QWL9=}3Z^+(<LJ%fi<d7r$4b>XZ<{u>39
zLw1j^<yFE#0vCd2^S5&_aHmKnc-5&T0OKcetV|##6=JEKC*^pVS;9%BM9j0|IJA>k
z9C_)zaTJb4iSblNT`i9EJyARroOUwa!|g}WN;1_II0@+qEC`Ymc~*+3|2j4vCuP*V
z0F4j7JS7e}rPuM%plgB&C``)`UHLs!#-{_B5GKTw0L>cpV;kz$9JBt$78Vu4rW)V7
zxSO3@KeNsPmgm0lh87rFuWx_;%g@hrb+?rgk{I-nl$FF=B<38n&D!~^vP5s1F&Bp!
zXv2TDhcwkbpO-(Rqy31gdsO|H4v7q6xfwc<xs-}W<(sc}!=C$GPmIJ@+dhN_^#Lkk
zz%F>F{&n3DG~Gw9&w5tw)iP)tBNuD8>$hF`3uqX)yOJnj;L1B_<Xw{Awmc)(Ae)B7
zzEu(RC_&)%@17;#I#mYcv_J}R%ZdChttWQ2_ZYS6EQ5QIL@)BEsG=+N;P!d!NIFP#
ztsYP7oOW-S^yGb-yIRk^eqWNZ$k`saJNY5?Z`9C>5nyjqZevywtI3~HMVZ0Uy=81m
th1MGv?tlne+T0VqkN;rPTzo)rtBB%je|O+x`*S<s)AL`u6E^S*z5&yb+O_}y

literal 0
HcmV?d00001

diff --git a/scripts/utils/__pycache__/vocab_utils.cpython-37.pyc b/scripts/utils/__pycache__/vocab_utils.cpython-37.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6c290f1d5af3e631a53de17140a2543863f10255
GIT binary patch
literal 1580
zcmYjR-EJc_6t+FTnWWinsVd6PDq40CS!tUakhnkyD0C4Q6|gH7jeub?c9KcvueQ^q
zVMapj-f*!X@dU{wufb#Z7HK7T1g<!C3dNF-eQclO&-VBEqn(`qLHX(3%kO`75&A<r
z{}u~6A3znKfuV@v429bkhuHWeB&M}O%d~cALrXFzcS9E=>QRf@&v58bhq}*D=u`X{
z_5D8~A3E)0<$P5p@i1|;55<Stf_e;9`~^m$HKurjF_L6zt?im@u(*^?jjnLxEFaYP
zWAyFa#x?86+O6HGx7n`uYe#!m1f#QiPf-m3XD$9+?W|k1RofdFmqAUabwz&JYdqOr
zcOb&k<4@s`U!X71*QlT-vkjSHXM;n>I1saIVs;<`klaL=eSH2qMYR)ki+i;*?M`n|
z_YnN)?#$vZAmeQcQE)xJcp>*LaqYmRv`$xddcEO1&1hPTdvPy|i}5@jvtC|OmK_}(
ztzPLtNejxBdc~sTRP;u?%zHmHiY$h!Jw2b5F=g~-oDVsMn_Ru4Cr5Fb!K5tBGLxdm
z@*$&U&jm}z6Df{*$%G}do}8rOr2mrgvT{6z9hIFHqf(JlD7%W~M7aP&NWlr@SMI3f
zc`TKm<#ZU)l&iLZ8X27Gw#H@p(K$<G$stKuu|m03n#?k$Y&y>?W$R_?K*vP?0gJAI
zDMw7=iYc$SPU<aUUT9?94~1Y{re*O7=Oyp^$}ZxZg&i8pSTGr708|A@3@3{;_Hf5+
z0Z_KCKVq>2)VT`uy+`KGm3JMk@aQ+hUxQ=*{o^15%4QEm!eg0C2Gz98*+sk>@KU5K
z8Z#m1oJB)cB$GVmvq6%T1*3!GY%V0@0Ctc=Z*W{bd|WY>Orl5EA&*$WASaw*0Ljxz
ziosl_nHZdFjYOt@R9&dH;Yegg+W<+t2?pT+cd!q35APBmcSr}nOm<;xA3PsM1MCxi
z2UaHfxpdLNAo;D@eua)iQ(S}ID5&DO)sUJ@3H0sfxUp($ZEa9vFF&ts(A=lrkj4Ss
z*=tZ4D8O63U4zIE5zM{S_{+PsuQ?CiLDQ;R)H_5?P$SS$pqYR6<S9Bkf%*WHN!H#5
zfg1gFi~2R11_#Kf0~u93Iq4JSa~9LG$SyQNl9~QF7Ce<KbdoHs1bA8uTcYo4q8sV~
z)9=%*PWo+#1Y+_ooKrbN2832_GUuY?%BwD7pv^+HDFYg%ah9%_3U27DV4D(V^gqS*
zmGONr{Wd?;eSMGNmiDYT4=uSY_<a~t){I>!Yd9Y%=RBpXRJKMNwhZ-*mTcWM#M+wI
zb-oT5{ifmo48r;mebU7N)Gj>b0X(Q#(<i5={ni$g2P^q4ZQjsEhllO$$6<(KG-{sT
jg#K?!&1$aqRsMSrFwfZg`o9qRqBL`S%^%XSf>-|m!2huO

literal 0
HcmV?d00001

diff --git a/scripts/utils/average_meter.py b/scripts/utils/average_meter.py
new file mode 100644
index 0000000..a9a4f61
--- /dev/null
+++ b/scripts/utils/average_meter.py
@@ -0,0 +1,23 @@
+
+class AverageMeter(object):
+    """Computes and stores the average and current value"""
+    def __init__(self, name, fmt=':f'):
+        self.name = name
+        self.fmt = fmt
+        self.reset()
+
+    def reset(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+
+    def update(self, val, n=1):
+        self.val = val
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+
+    def __str__(self):
+        fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'
+        return fmtstr.format(**self.__dict__)
diff --git a/scripts/utils/data_utils.py b/scripts/utils/data_utils.py
new file mode 100644
index 0000000..4545e27
--- /dev/null
+++ b/scripts/utils/data_utils.py
@@ -0,0 +1,48 @@
+import json
+import re
+
+
+def normalize_string(s):
+    """ lowercase, trim, and remove non-letter characters """
+    s = s.lower().strip()
+    s = re.sub(r"([,.!?])", r" \1 ", s)  # isolate some marks
+    s = re.sub(r"(['])", r"", s)  # remove apostrophe (i.e., shouldn't --> shouldnt)
+    s = re.sub(r"[^a-zA-Z0-9,.!?]+", r" ", s)  # replace other characters with whitespace
+    s = re.sub(r"\s+", r" ", s).strip()
+    return s
+
+
+class SubtitleWrapper:
+    TIMESTAMP_PATTERN = re.compile('(\d+)?:?(\d{2}):(\d{2})[.,](\d{3})')
+
+    def __init__(self, subtitle_path):
+        self.subtitle = []
+        self.load_gentle_subtitle(subtitle_path)
+
+    def get(self):
+        return self.subtitle
+
+    # using gentle lib
+    def load_gentle_subtitle(self, subtitle_path):
+        try:
+            with open(subtitle_path) as data_file:
+                data = json.load(data_file)
+                for item in data:
+                    if 'words' in item['alternatives'][0]:
+                        raw_subtitle = item['alternatives'][0]['words']
+                        for word in raw_subtitle:
+                            self.subtitle.append(word)
+        except FileNotFoundError:
+            self.subtitle = None
+
+    # convert timestamp to second
+    def get_seconds(self, word_time_e):
+        time_value = re.match(self.TIMESTAMP_PATTERN, word_time_e)
+        if not time_value:
+            print('wrong time stamp pattern')
+            exit()
+
+        values = list(map(lambda x: int(x) if x else 0, time_value.groups()))
+        hours, minutes, seconds, milliseconds = values[0], values[1], values[2], values[3]
+
+        return hours * 3600 + minutes * 60 + seconds + milliseconds / 1000
diff --git a/scripts/utils/data_utils_twh.py b/scripts/utils/data_utils_twh.py
new file mode 100644
index 0000000..b83bcec
--- /dev/null
+++ b/scripts/utils/data_utils_twh.py
@@ -0,0 +1,65 @@
+import csv
+import json
+import re
+
+
+def normalize_string(s):
+    """ lowercase, trim, and remove non-letter characters """
+    s = s.lower().strip()
+    # s = re.sub(r"([,.!?])", r" \1 ", s)  # isolate some marks
+    s = re.sub(r"([,.!?])", r"", s)  # remove marks
+    s = re.sub(r"(['])", r"", s)  # remove apostrophe (i.e., shouldn't --> shouldnt)
+    s = re.sub(r"[^a-zA-Z0-9,.!?]+", r" ", s)  # replace other characters with whitespace
+    s = re.sub(r"\s+", r" ", s).strip()
+    return s
+
+
+class SubtitleWrapper:
+    TIMESTAMP_PATTERN = re.compile('(\d+)?:?(\d{2}):(\d{2})[.,](\d{3})')
+
+    def __init__(self, subtitle_path):
+        self.subtitle = []
+        self.load_tsv_subtitle(subtitle_path)
+
+    def get(self):
+        return self.subtitle
+
+    def load_tsv_subtitle(self, subtitle_path):
+        try:
+            with open(subtitle_path) as file:
+
+                # I had to update it since file number 157 has a different structure and make some problems
+
+                for line in file:
+                    line = line.strip()
+                    if line.__contains__("content\t\t"):
+                        line = line[len('content\t\t'):]
+                        print("*****************************Error Content")
+                    splitted = line.split("\t")
+                    if len(splitted)==2:
+                        splitted.append("eh")
+                        print("*****************************Error Lost Word " + splitted[0])
+
+                    self.subtitle.append(splitted)
+
+                # file.seek(0)
+                # tsv_file = csv.reader(file, delimiter="\t")
+                # for line in tsv_file:
+                #     self.subtitle.append(line)
+
+
+        except FileNotFoundError:
+            self.subtitle = None
+        print()
+
+    # convert timestamp to second
+    def get_seconds(self, word_time_e):
+        time_value = re.match(self.TIMESTAMP_PATTERN, word_time_e)
+        if not time_value:
+            print('wrong time stamp pattern')
+            exit()
+
+        values = list(map(lambda x: int(x) if x else 0, time_value.groups()))
+        hours, minutes, seconds, milliseconds = values[0], values[1], values[2], values[3]
+
+        return hours * 3600 + minutes * 60 + seconds + milliseconds / 1000
diff --git a/scripts/utils/train_utils.py b/scripts/utils/train_utils.py
new file mode 100644
index 0000000..ddc6159
--- /dev/null
+++ b/scripts/utils/train_utils.py
@@ -0,0 +1,85 @@
+import logging
+import os
+import random
+from logging.handlers import RotatingFileHandler
+
+import numpy as np
+import time
+import math
+import torch
+
+from train_DAE import init_model as DAE_init
+from train_Autoencoder import init_model as autoencoder_init
+from train_cluster2gesture import init_model as c2g_init
+from train_text2embedding import init_model as text2embedding_init
+from train_gan import init_model as text2embedding_gan_init
+from train_autoencoder_VQVAE import init_model as VQVAE_init
+from train import init_model as baseline_init_model
+def set_logger(log_path=None, log_filename='log'):
+    for handler in logging.root.handlers[:]:
+        logging.root.removeHandler(handler)
+    handlers = [logging.StreamHandler()]
+    if log_path is not None:
+        os.makedirs(log_path, exist_ok=True)
+        handlers.append(
+            RotatingFileHandler(os.path.join(log_path, log_filename), maxBytes=10 * 1024 * 1024, backupCount=5))
+    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s: %(message)s', handlers=handlers)
+    logging.getLogger("matplotlib").setLevel(logging.WARNING)
+
+
+def as_minutes(s):
+    m = math.floor(s / 60)
+    s -= m * 60
+    return '%dm %ds' % (m, s)
+
+
+def time_since(since):
+    now = time.time()
+    s = now - since
+    return '%s' % as_minutes(s)
+
+
+def save_checkpoint(state, filename):
+    torch.save(state, filename)
+    logging.info('Saved the checkpoint')
+
+
+def load_checkpoint_and_model(checkpoint_path, _device='cpu', what=''):
+    print('loading checkpoint {}'.format(checkpoint_path))
+    checkpoint = torch.load(checkpoint_path, map_location=_device)
+    args = checkpoint['args']
+    epoch = checkpoint['epoch']
+    lang_model = checkpoint['lang_model']
+    pose_dim = checkpoint['pose_dim']
+    print('epoch {}'.format(epoch))
+
+    if what == 'c2g':
+        generator, loss_fn = c2g_init(args, lang_model, pose_dim, _device)
+        generator.load_state_dict(checkpoint['gen_dict'])
+    elif what == 'text2embedding':
+        generator, loss_fn = text2embedding_init(args, lang_model, pose_dim, _device)
+        generator.load_state_dict(checkpoint['gen_dict'])
+        # Todo: should change to what==? and then fix all it's callings.
+    elif what == 'DAE': #Todo
+        generator, loss_fn = DAE_init(args, lang_model, pose_dim, _device)
+        generator.load_state_dict(checkpoint['gen_dict'])
+    elif what == "autoencoder": #Todo
+        generator, loss_fn = autoencoder_init(args, lang_model, pose_dim, _device)
+        generator.load_state_dict(checkpoint['gen_dict'])
+    elif what == 'baseline':
+        generator, loss_fn = baseline_init_model(args, lang_model, pose_dim, _device)
+        generator.load_state_dict(checkpoint['gen_dict'])
+    elif what == 'text2embedding_gan':
+        generator, loss_fn = text2embedding_gan_init(args, lang_model, pose_dim, _device)
+        generator.load_state_dict(checkpoint['gen_dict'])
+    elif what == 'autoencoder_vq':
+        generator, loss_fn = VQVAE_init(args, lang_model, pose_dim, _device)
+        generator.load_state_dict(checkpoint['gen_dict'])
+    else:
+        assert (1==2)
+
+
+    # set to eval mode
+    generator.train(False)
+
+    return args, generator, loss_fn, lang_model, pose_dim
diff --git a/scripts/utils/vocab_utils.py b/scripts/utils/vocab_utils.py
new file mode 100644
index 0000000..bd55d0f
--- /dev/null
+++ b/scripts/utils/vocab_utils.py
@@ -0,0 +1,57 @@
+import logging
+import os
+import pickle
+
+import lmdb
+import pyarrow
+
+from model.vocab import Vocab
+
+
+def build_vocab(name, dataset_list, cache_path, word_vec_path=None, feat_dim=None):
+    logging.info('  building a language model...')
+    if not os.path.exists(cache_path):
+        lang_model = Vocab(name)
+        for dataset in dataset_list:
+            logging.info('    indexing words from {}'.format(dataset.lmdb_dir))
+            index_words(lang_model, dataset.lmdb_dir)
+
+        if word_vec_path is not None:
+            lang_model.load_word_vectors(word_vec_path, feat_dim)
+
+        with open(cache_path, 'wb') as f:
+            pickle.dump(lang_model, f)
+    else:
+        logging.info('    loaded from {}'.format(cache_path))
+        with open(cache_path, 'rb') as f:
+            lang_model = pickle.load(f)
+
+        if word_vec_path is None:
+            lang_model.word_embedding_weights = None
+        elif lang_model.word_embedding_weights.shape[0] != lang_model.n_words:
+            logging.warning('    failed to load word embedding weights. check this')
+            assert False
+
+    return lang_model
+
+
+def index_words(lang_model, lmdb_dir):
+    lmdb_env = lmdb.open(lmdb_dir, readonly=True, lock=False)
+    txn = lmdb_env.begin(write=False)
+    cursor = txn.cursor()
+
+    for key, buf in cursor:
+        video = pyarrow.deserialize(buf)
+
+        for clip in video['clips']:
+            for word_info in clip['words']:
+                word = word_info[0]
+                lang_model.index_word(word)
+
+    lmdb_env.close()
+    logging.info('    indexed %d words' % lang_model.n_words)
+
+    # filtering vocab
+    # MIN_COUNT = 3
+    # lang_model.trim(MIN_COUNT)
+