06_encode_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from models import *\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define helper functions\n",
    "def save_encoding(data, data_anno, path_vae, path_pca, path_kpca_cosine, path_kpca_sigmoid, path_kpca_rbf, path_kpca_poly):\n",
    "    vae_enc = mtmodel.encode_mu(data.values)\n",
    "    vae_enc = pd.DataFrame(vae_enc)\n",
    "    vae_enc = pd.concat([data_anno, vae_enc], axis = 1)\n",
    "    # rename dimension 0 to 18\n",
    "    vae_enc = vae_enc.rename(columns={0: 18})\n",
    "    #  move dimension 18 to the last column\n",
    "    vae_enc = vae_enc[ [ col for col in vae_enc.columns if col != 18 ] + [18] ]\n",
    "    vae_enc.to_csv(path_vae, index = False)\n",
    "    \n",
    "    pca_enc = PCA_model_.encode(data)\n",
    "    pca_enc = pd.DataFrame(pca_enc)\n",
    "    pca_enc = pd.concat([data_anno, pca_enc], axis = 1)\n",
    "    # Dimensions start at 0 (form python index 0). Add\n",
    "    # 1 to each dimension name to start with Dimension 1\n",
    "    pca_enc = pca_enc.rename(columns={i : i+1 for i in range(18)})\n",
    "    pca_enc.to_csv(path_pca, index = False)\n",
    "    \n",
    "    kpca_cosine_enc = KPCA_cosine_model_.encode(data)\n",
    "    kpca_cosine_enc = pd.DataFrame(kpca_cosine_enc)\n",
    "    kpca_cosine_enc = pd.concat([data_anno, kpca_cosine_enc], axis = 1)\n",
    "    # Dimensions start at 0 (form python index 0). Add\n",
    "    # 1 to each dimension name to start with Dimension 1\n",
    "    kpca_cosine_enc = kpca_cosine_enc.rename(columns={i : i+1 for i in range(18)})\n",
    "    kpca_cosine_enc.to_csv(path_kpca_cosine, index = False)\n",
    "    \n",
    "    kpca_sigmoid_enc = KPCA_sigmoid_model_.encode(data)\n",
    "    kpca_sigmoid_enc = pd.DataFrame(kpca_sigmoid_enc)\n",
    "    kpca_sigmoid_enc = pd.concat([data_anno, kpca_sigmoid_enc], axis = 1)\n",
    "    # Dimensions start at 0 (form python index 0). Add\n",
    "    # 1 to each dimension name to start with Dimension 1\n",
    "    kpca_sigmoid_enc = kpca_sigmoid_enc.rename(columns={i : i+1 for i in range(18)})\n",
    "    kpca_sigmoid_enc.to_csv(path_kpca_sigmoid, index = False)\n",
    "    \n",
    "    kpca_rbf_enc = KPCA_rbf_model_.encode(data)\n",
    "    kpca_rbf_enc = pd.DataFrame(kpca_rbf_enc)\n",
    "    kpca_rbf_enc = pd.concat([data_anno, kpca_rbf_enc], axis = 1)\n",
    "    # Dimensions start at 0 (form python index 0). Add\n",
    "    # 1 to each dimension name to start with Dimension 1\n",
    "    kpca_rbf_enc = kpca_rbf_enc.rename(columns={i : i+1 for i in range(18)})\n",
    "    kpca_rbf_enc.to_csv(path_kpca_rbf, index = False)\n",
    "    \n",
    "    kpca_poly_enc = KPCA_poly_model_.encode(data)\n",
    "    kpca_poly_enc = pd.DataFrame(kpca_poly_enc)\n",
    "    kpca_poly_enc = pd.concat([data_anno, kpca_poly_enc], axis = 1)\n",
    "    # Dimensions start at 0 (form python index 0). Add\n",
    "    # 1 to each dimension name to start with Dimension 1\n",
    "    kpca_poly_enc = kpca_poly_enc.rename(columns={i : i+1 for i in range(18)})\n",
    "    kpca_poly_enc.to_csv(path_kpca_poly, index = False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model and encoding path\n",
    "model_path = 'models/'\n",
    "encode_path = 'results/encodings/'\n",
    "\n",
    "# Saved VAE model paths\n",
    "path_vae     = model_path + 'VAE.h5'\n",
    "path_encoder = model_path + 'VAE_encoder.h5'\n",
    "path_decoder = model_path + 'VAE_decoder.h5'\n",
    "\n",
    "# Save paths for encoded data\n",
    "qm_vae_encoding = encode_path + 'QMDiab_VAE_encoding.csv'\n",
    "qm_pca_encoding = encode_path + 'QMDiab_PCA_encoding.csv'\n",
    "qm_kpca_cosine_encoding = encode_path + 'QMDiab_KPCA_cosine_encoding.csv'\n",
    "qm_kpca_sigmoid_encoding = encode_path + 'QMDiab_KPCA_sigmoid_encoding.csv'\n",
    "qm_kpca_rbf_encoding = encode_path + 'QMDiab_KPCA_rbf_encoding.csv'\n",
    "qm_kpca_poly_encoding = encode_path + 'QMDiab_KPCA_poly_encoding.csv'\n",
    "\n",
    "aml_vae_encoding = encode_path + 'AML_VAE_encoding.csv'\n",
    "aml_pca_encoding = encode_path + 'AML_PCA_encoding.csv'\n",
    "aml_kpca_cosine_encoding = encode_path + 'AML_KPCA_cosine_encoding.csv'\n",
    "aml_kpca_sigmoid_encoding = encode_path + 'AML_KPCA_sigmoid_encoding.csv'\n",
    "aml_kpca_rbf_encoding = encode_path + 'AML_KPCA_rbf_encoding.csv'\n",
    "aml_kpca_poly_encoding = encode_path + 'AML_KPCA_poly_encoding.csv'\n",
    "\n",
    "schizo_vae_encoding = encode_path + 'Schizo_VAE_encoding.csv'\n",
    "schizo_pca_encoding = encode_path + 'Schizo_PCA_encoding.csv'\n",
    "schizo_kpca_cosine_encoding = encode_path + 'Schizo_KPCA_cosine_encoding.csv'\n",
    "schizo_kpca_sigmoid_encoding = encode_path + 'Schizo_KPCA_sigmoid_encoding.csv'\n",
    "schizo_kpca_rbf_encoding = encode_path + 'Schizo_KPCA_rbf_encoding.csv'\n",
    "schizo_kpca_poly_encoding = encode_path + 'Schizo_KPCA_poly_encoding.csv'\n",
    "\n",
    "# Data paths\n",
    "twins_path = 'data/TwinsUK.xls'\n",
    "qmdiab_path = 'data/QMDiab.xls'\n",
    "aml_path = 'data/AML.xls'\n",
    "schizo_path = 'data/Schizophrenia.xls'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load datasets\n",
    "twins_train_df = pd.read_excel(twins_path, sheet_name='Training Set')\n",
    "twins_test_df  = pd.read_excel(twins_path, sheet_name='Testing Set')\n",
    "twins_full_data = pd.concat([twins_train_df, twins_test_df], ignore_index = True)\n",
    "\n",
    "qmdiab_data = pd.read_excel(qmdiab_path, sheet_name='Metabolite Data')\n",
    "qmdiab_anno = pd.read_excel(qmdiab_path, sheet_name='Sample Annotations')\n",
    "\n",
    "aml_data = pd.read_excel(aml_path, sheet_name='Metabolite Data')\n",
    "aml_anno = pd.read_excel(aml_path, sheet_name='Sample Annotations')\n",
    "\n",
    "schizo_data = pd.read_excel(schizo_path, sheet_name='Metabolite Data')\n",
    "schizo_anno = pd.read_excel(schizo_path, sheet_name='Sample Annotations')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data & model configuration\n",
    "input_dim = twins_train_df.shape[1]\n",
    "intermediate_dim = 200\n",
    "latent_dim = 18\n",
    "\n",
    "kl_beta = 1e-2\n",
    "learning_rate = 1e-3\n",
    "\n",
    "# Instantiate model\n",
    "mtmodel = mtVAE(input_dim,\n",
    "                intermediate_dim,\n",
    "                latent_dim,\n",
    "                kl_beta,\n",
    "                learning_rate)\n",
    "\n",
    "\n",
    "# Load VAE model\n",
    "mtmodel.vae.load_weights(path_vae)\n",
    "mtmodel.encoder.load_weights(path_encoder)\n",
    "mtmodel.decoder.load_weights(path_decoder)\n",
    "\n",
    "\n",
    "######################\n",
    "# Define PCA model\n",
    "######################\n",
    "PCA_model_ = PCA_model(twins_train_df.values, latent_dim)\n",
    "\n",
    "######################\n",
    "# Define KPCA models\n",
    "######################\n",
    "KPCA_cosine_model_ = KPCA_model(twins_train_df.values, latent_dim, \"cosine\", 1, 0, 0, 0)\n",
    "KPCA_sigmoid_model_ = KPCA_model(twins_train_df.values, latent_dim, \"sigmoid\", 1, 0.05, 0, 0)\n",
    "KPCA_rbf_model_ = KPCA_model(twins_train_df.values, latent_dim, \"rbf\", 1, 0.005, 0, 0)\n",
    "KPCA_poly_model_ = KPCA_model(twins_train_df.values, latent_dim, \"poly\", 2, 0.001, 3, 5.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate and save data encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# QMDiab encoding\n",
    "save_encoding(qmdiab_data, qmdiab_anno, qm_vae_encoding, qm_pca_encoding, qm_kpca_cosine_encoding, qm_kpca_sigmoid_encoding, qm_kpca_rbf_encoding, qm_kpca_poly_encoding)\n",
    "\n",
    "# AML encoding\n",
    "save_encoding(aml_data, aml_anno, aml_vae_encoding, aml_pca_encoding, aml_kpca_cosine_encoding, aml_kpca_sigmoid_encoding, aml_kpca_rbf_encoding, aml_kpca_poly_encoding)\n",
    "\n",
    "# Schizo encoding\n",
    "save_encoding(schizo_data, schizo_anno, schizo_vae_encoding, schizo_pca_encoding, schizo_kpca_cosine_encoding, schizo_kpca_sigmoid_encoding, schizo_kpca_rbf_encoding, schizo_kpca_poly_encoding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}