LA4DS_ch15.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "colab": {
   "name": "LA4DS_ch15.ipynb",
   "provenance": [
    {
     "file_id": "1BJns485WqoiFl8sauu7Q1vkwAmQy4qwo",
     "timestamp": 1651231889273
    },
    {
     "file_id": "1qFviZiAMrt-M__X5-4FTU77UKZKCAWnU",
     "timestamp": 1643017491021
    }
   ],
   "collapsed_sections": [],
   "authorship_tag": "ABX9TyPZi0KD+UMopqd/pJIAWGig"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Practical Linear Algebra for Data Science\n",
    "## Mike X Cohen (sincxpress.com)\n",
    "### https://www.oreilly.com/library/view/practical-linear-algebra/9781098120603/\n",
    "\n",
    "#### Code for chapter 15"
   ],
   "metadata": {
    "id": "SbGFWGzkd44U"
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pq-JEewsnKHm"
   },
   "outputs": [],
   "source": [
    ""
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.gridspec import GridSpec # for the subplots\n",
    "\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "# NOTE: these lines define global figure properties used for publication.\n",
    "import matplotlib_inline.backend_inline\n",
    "matplotlib_inline.backend_inline.set_matplotlib_formats('svg') # display figures in vector format\n",
    "plt.rcParams.update({'font.size':14}) # set global font size"
   ],
   "metadata": {
    "id": "GVWZlfT-nThD"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "dxDGYYj19deH"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Creating Figure 1"
   ],
   "metadata": {
    "id": "LAALGfJV9dho"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# Create some correlated data\n",
    "X = np.random.randn(1000,2)\n",
    "X[:,1] = np.sum(X,axis=1)\n",
    "\n",
    "# quick PCA\n",
    "evals,evecs = np.linalg.eig( np.cov(X.T,ddof=1) )\n",
    "scores = X @ evecs\n",
    "\n",
    "\n",
    "# show in a plot\n",
    "_,axs = plt.subplots(1,2,figsize=(10,5))\n",
    "axs[0].plot(X[:,0],X[:,1],'ko',markerfacecolor='w')\n",
    "axs[0].plot([0,3*evecs[0,1]],[0,3*evecs[1,1]],'r-',linewidth=4,label='PC1')\n",
    "axs[0].plot([0,3*evecs[0,0]],[0,3*evecs[1,0]],'r:',linewidth=4,label='PC2')\n",
    "axs[0].axis([-5,5,-5,5])\n",
    "axs[0].set_xlabel('Data axis 1')\n",
    "axs[0].set_ylabel('Data axis 2')\n",
    "axs[0].legend()\n",
    "axs[0].set_title('Data in channel space')\n",
    "\n",
    "\n",
    "axs[1].plot(scores[:,1],scores[:,0],'ko',markerfacecolor='w')\n",
    "axs[1].set_xlabel('PC axis 1')\n",
    "axs[1].set_ylabel('PC axis 2')\n",
    "axs[1].axis([-6,6,-6,6])\n",
    "axs[1].set_title('Data in PC space')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('Figure_15_01.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "aXAxFYVG9gjY"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# Empirical demonstration that variance and squared vector norm are equal.\n",
    "# You can prove their equivalence by writing down their formulas and assuming the vector is mean-centered.\n",
    "\n",
    "# extract one variable\n",
    "q = X[:,1]\n",
    "\n",
    "# compute variance\n",
    "var = np.var(q,ddof=1)\n",
    "\n",
    "# compute squared vector norm (after mean-centering)\n",
    "norm = np.linalg.norm( q-np.mean(q) )**2\n",
    "\n",
    "# show that they're the same (with the scaling factor)\n",
    "print(var)\n",
    "print(norm / (len(q)-1))"
   ],
   "metadata": {
    "id": "eBTEo18K9dvz"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "kJ7Qwb6xK9jt"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Exercise 1 (PCA of Instanbul stock exchange)"
   ],
   "metadata": {
    "id": "mWvZfl5qjyhT"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# Data citation: Akbilgic, Oguz. (2013). ISTANBUL STOCK EXCHANGE. UCI Machine Learning Repository.\n",
    "# data source website: https://archive-beta.ics.uci.edu/ml/datasets/istanbul+stock+exchange\n",
    "\n",
    "# import the data\n",
    "url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/00247/data_akbilgic.xlsx\"\n",
    "data = pd.read_excel(url,index_col=0,skiprows=1)\n",
    "\n",
    "# let's have a look\n",
    "data"
   ],
   "metadata": {
    "id": "g8oFPIQg1BGI"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# show some data in line plots\n",
    "data.plot(figsize=(15,6),ylabel='Market returns')\n",
    "plt.savefig('Figure_15_03a.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "QeRsvLyA1BJH"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# Seaborn's pairplot shows a lot of positive correlations\n",
    "# I don't show this in the book b/c it's too big, lol.\n",
    "sns.pairplot(data,height=1.5)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "dMTlRjpkRGA3"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "### show the correlation matrix in an image\n",
    "\n",
    "plt.figure(figsize=(8,8))\n",
    "heatmap = sns.heatmap(data.corr(),vmin=-1,vmax=1,annot=True,cmap='bwr')\n",
    "plt.savefig('Figure_15_03b.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "iqg7VoNCRarv"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "d_F0V271e2Qo"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "#### now for PCA!\n",
    "\n",
    "# Step 1: covariance matrix\n",
    "X = data.values # extract data\n",
    "X = X - np.mean(X,axis=0,keepdims=True) # mean-center via broadcasting\n",
    "\n",
    "# note: these data are observations-by-features, so we need X'X, not XX'\n",
    "covmat = X.T@X / (X.shape[0]-1)\n",
    "\n",
    "# visualize it\n",
    "plt.figure(figsize=(6,6))\n",
    "plt.imshow(covmat,vmin=-.0002,vmax=.0002)\n",
    "plt.colorbar(shrink=.82)\n",
    "plt.title('Data covariance')\n",
    "plt.xticks(range(X.shape[1]),labels=data.columns,rotation=90)\n",
    "plt.yticks(range(X.shape[1]),labels=data.columns)\n",
    "plt.savefig('Figure_15_03c.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "AJWBg6Kh4Kd6"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# Step 2: eigendecomposition\n",
    "evals,evecs = np.linalg.eig(covmat)\n",
    "\n",
    "# Step 3: sort results\n",
    "sidx  = np.argsort(evals)[::-1]\n",
    "evals = evals[sidx]\n",
    "evecs = evecs[:,sidx]\n",
    "\n",
    "\n",
    "# Step 4: component scores\n",
    "components = data.values @ evecs[:,0:2]\n",
    "print(components.shape)\n",
    "\n",
    "# Step 5: eigenvalues to %var\n",
    "factorScores = 100*evals/np.sum(evals)\n",
    "\n",
    "\n",
    "# show scree plot\n",
    "plt.figure(figsize=(8,4))\n",
    "plt.plot(factorScores,'ks-',markersize=15)\n",
    "plt.xlabel('Component index')\n",
    "plt.ylabel('Percent variance')\n",
    "plt.title('Scree plot of stocks dataset')\n",
    "plt.grid()\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "1knqrtoQsOEM"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# Show that variance of the components equals the eigenvalue\n",
    "print('Variance of first two components:')\n",
    "print(np.var(components,axis=0,ddof=1)) # note the ddof=1! The default produces the biased variance.\n",
    "\n",
    "print(f'\\nFirst two eigenvalues:')\n",
    "print(evals[:2])"
   ],
   "metadata": {
    "id": "HUYMtMbW6BkA"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# correlate first two components\n",
    "\n",
    "plt.figure(figsize=(12,6))\n",
    "plt.plot(components)\n",
    "plt.xlabel('Time (day)')\n",
    "plt.legend(['Comp. 1','Comp. 2'])\n",
    "plt.title(f'Correlation r={np.corrcoef(components.T)[0,1]:.5f}')\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "2pcOSsn06Bdx"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "_,axs = plt.subplots(1,2,figsize=(12,5))\n",
    "\n",
    "for i in range(2):\n",
    "  axs[i].bar(range(X.shape[1]),evecs[:,i],color='black')\n",
    "  axs[i].set_xticks(range(X.shape[1]))\n",
    "  axs[i].set_xticklabels(data.columns,rotation=45)\n",
    "  axs[i].set_ylabel('Weight')\n",
    "  axs[i].set_title(f'Weights for component {i}')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "xzzOY3Wz6Bg8"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# Now all in one figure\n",
    "\n",
    "fig = plt.figure(figsize=(10,6))\n",
    "gs = GridSpec(2,4,figure=fig)\n",
    "\n",
    "# scree plot\n",
    "ax1 = fig.add_subplot(gs[0,0])\n",
    "ax1.plot(factorScores,'ks-',markersize=10)\n",
    "ax1.set_xlabel('Component index')\n",
    "ax1.set_ylabel('Percent variance')\n",
    "ax1.set_title('Scree plot')\n",
    "ax1.grid()\n",
    "\n",
    "\n",
    "# component time series\n",
    "ax2 = fig.add_subplot(gs[0,1:])\n",
    "ax2.plot(components)\n",
    "ax2.set_xlabel('Time (day)')\n",
    "ax2.set_xlim([0,components.shape[0]])\n",
    "ax2.legend(['Comp. 1','Comp. 2'])\n",
    "ax2.set_title(f'Correlation r={np.corrcoef(components.T)[0,1]:.5f}')\n",
    "\n",
    "\n",
    "# bar plots of component loadings\n",
    "axs = fig.add_subplot(gs[1,:2]), fig.add_subplot(gs[1,2:])\n",
    "for i in range(2):\n",
    "  axs[i].bar(range(X.shape[1]),evecs[:,i],color='black')\n",
    "  axs[i].set_xticks(range(X.shape[1]))\n",
    "  axs[i].set_xticklabels(data.columns,rotation=45)\n",
    "  axs[i].set_ylabel('Weight')\n",
    "  axs[i].set_title(f'Weights for component {i}')\n",
    "\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('Figure_15_04.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "tl_vgspIJpXN"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "YEV-bPEtBcbp"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Exercise 2"
   ],
   "metadata": {
    "id": "nZu7QGcxVU3_"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "### SVD on covariance matrix\n",
    "\n",
    "# It suffices to show that the eigenvalues and singular values match, and that the eigenvectors and singular vectors match.\n",
    "# Here I only show the first four values and the first vector.\n",
    "\n",
    "# SVD\n",
    "U,s,Vt = np.linalg.svd(covmat)\n",
    "\n",
    "# eigen/singular values\n",
    "print('First 4 eigenvalues:')\n",
    "print(evals[:4])\n",
    "\n",
    "print(f'\\nFirst 4 singular values:')\n",
    "print(s[:4])\n",
    "\n",
    "\n",
    "# eigen/singular vectors\n",
    "print('\\n\\n\\nFirst eigenvector:')\n",
    "print(evecs[:,0])\n",
    "\n",
    "print('\\nFirst singular vector:')\n",
    "print(U[:,0])"
   ],
   "metadata": {
    "id": "ktIVCSQyVYUB"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "### SVD on data matrix\n",
    "\n",
    "# Again, we can simply show that the singular values (suitably normalized) match the eigenvalues, and that\n",
    "# the singular vectors match the eigenvectors.\n",
    "\n",
    "# Note that the data variable X is already mean-centered!\n",
    "U,s,Vt = np.linalg.svd(X)  # SVD\n",
    "\n",
    "\n",
    "# eigen/singular values\n",
    "print('First 4 eigenvalues:')\n",
    "print(evals[:4])\n",
    "\n",
    "print(f'\\nFirst 4 singular values:')\n",
    "print(s[:4]**2/(X.shape[0]-1))\n",
    "\n",
    "\n",
    "# eigen/singular vectors\n",
    "print('\\n\\n\\nFirst eigenvector:')\n",
    "print(evecs[:,0])\n",
    "\n",
    "print('\\nFirst right singular vector:')\n",
    "print(Vt[0,:])"
   ],
   "metadata": {
    "id": "k_q3kUotVYZB"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "jJQr8NYvVYb-"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Exercise 3"
   ],
   "metadata": {
    "id": "k85ZlCd6VYe_"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "### As above, it suffices to show that the eigenvalues and eigenvectors match.\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    " \n",
    "pca = PCA()\n",
    "X_t = pca.fit_transform(data)\n",
    "\n",
    "# compare percent-normalized eigenvalues\n",
    "print('Eigenvalues:')\n",
    "print(evals[:4])\n",
    "\n",
    "print(f'\\nExplained variance from sklearn:')\n",
    "print(pca.explained_variance_[:4])\n",
    "\n",
    "\n",
    "\n",
    "# eigenvector and sklearn component\n",
    "print('\\n\\n\\nFirst eigenvector:')\n",
    "print(evecs[:,0])\n",
    "\n",
    "print('\\nFirst sklearn component vector:')\n",
    "print(pca.components_[0,:])"
   ],
   "metadata": {
    "id": "JJngClhjVYhs"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "5IOxrtAkVU7G"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Exercise 4"
   ],
   "metadata": {
    "id": "90mzaLpWVU-N"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# generate data\n",
    "\n",
    "x = np.hstack((np.random.randn(1000,1),.05*np.random.randn(1000,1)))\n",
    "\n",
    "# rotation matrices\n",
    "th = -np.pi/6\n",
    "R1 = np.array([ [np.cos(th), -np.sin(th)],\n",
    "                [np.sin(th),  np.cos(th)] ])\n",
    "th = -np.pi/3\n",
    "R2 = np.array([ [np.cos(th), -np.sin(th)],\n",
    "                [np.sin(th),  np.cos(th)] ])\n",
    "\n",
    "# create the data\n",
    "X = np.vstack((x@R1,x@R2))\n",
    "X.shape"
   ],
   "metadata": {
    "id": "uOXRtupjVVBF"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# PCA via SVD\n",
    "U,s,Vt = np.linalg.svd(X-np.mean(X,axis=0,keepdims=True))\n",
    "\n",
    "# not necessary: convert singular values into eigenvalues\n",
    "s = s**2 / (X.shape[0]-1)\n",
    "\n",
    "# also not necessary: up-scale the singular vectors for visualization\n",
    "Vt *= 2"
   ],
   "metadata": {
    "id": "IJOwhYybVVD5"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# plot the data and eigenvectors\n",
    "\n",
    "plt.figure(figsize=(7,7))\n",
    "\n",
    "# the data\n",
    "plt.plot(X[:,0],X[:,1],'ko',markerfacecolor='w')\n",
    "\n",
    "# eigenvectors\n",
    "plt.plot([0,Vt[0,0]],[0,Vt[1,0]],'r--',linewidth=5,label='Comp 1')\n",
    "plt.plot([0,Vt[0,1]],[0,Vt[1,1]],'r:',linewidth=5,label='Comp 2')\n",
    "\n",
    "plt.legend()\n",
    "plt.grid()\n",
    "plt.savefig('Figure_15_05.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "8r3dHB3BVVGx"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "RhonrJwJVVJ1"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Exercise 5"
   ],
   "metadata": {
    "id": "tM8JXK6vYJ_c"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# create the data\n",
    "N = 200\n",
    "\n",
    "class1 = np.random.randn(N,2)\n",
    "class1[:,1] += class1[:,0]\n",
    "class1 += np.array([2,-1])\n",
    "\n",
    "class2 = np.random.randn(N,2)\n",
    "class2[:,1] += class2[:,0]\n",
    "\n",
    "# for later, it will be convenient to have the data in one matrix\n",
    "alldata = np.vstack((class1,class2))\n",
    "labels  = np.append(np.zeros(N),np.ones(N))\n",
    "\n",
    "\n",
    "\n",
    "# show data in their original data space\n",
    "ax = sns.jointplot(x=alldata[:,0],y=alldata[:,1],hue=labels)\n",
    "ax.ax_joint.set_xlabel('Data axis 1')\n",
    "ax.ax_joint.set_ylabel('Data axis 2')\n",
    "ax.plot_joint(sns.kdeplot)\n",
    "plt.savefig('Figure_15_02a.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "FJgPlQOeYNm2"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "hf4MXu4kNa-A"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Exercise 6"
   ],
   "metadata": {
    "id": "6kNd1q-kNbR1"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# LDA\n",
    "\n",
    "# between-class covariance\n",
    "cmc1 = np.mean(class1,axis=0)\n",
    "cmc2 = np.mean(class2,axis=0)\n",
    "covB = np.cov(np.vstack((cmc1,cmc2)).T,ddof=1)\n",
    "\n",
    "# within-class covariances\n",
    "cov1 = np.cov(class1.T,ddof=1)\n",
    "cov2 = np.cov(class2.T,ddof=1)\n",
    "covW = (cov1+cov2)/2\n",
    "\n",
    "\n",
    "# LDA via GED\n",
    "from scipy.linalg import eigh\n",
    "evals,evecs = eigh(covB,covW)\n",
    "\n",
    "# sort the solution\n",
    "sidx  = np.argsort(evals)[::-1]\n",
    "evals = evals[sidx]\n",
    "evecs = evecs[:,sidx]\n",
    "\n",
    "\n",
    "# project the mean-centered data onto the GED axes\n",
    "projA = (alldata-np.mean(alldata,axis=0)) @ evecs  # A=all"
   ],
   "metadata": {
    "id": "Dtpkwj1lYNsL"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# show the data\n",
    "_,axs = plt.subplots(1,2,figsize=(12,6))\n",
    "marker = ['bo','r+']\n",
    "for i in range(2):\n",
    "  axs[0].plot(alldata[labels==i,0],alldata[labels==i,1],marker[i],label=f'Class {i}')\n",
    "\n",
    "axs[0].plot([0,evecs[0,0]],[0,evecs[1,0]],'k-',linewidth=3,label='C1')\n",
    "axs[0].plot([0,evecs[0,1]],[0,evecs[1,1]],'k:',linewidth=3,label='C2')\n",
    "axs[0].set_xlabel('Data axis 1')\n",
    "axs[0].set_ylabel('Data axis 2')\n",
    "axs[0].set_title('Data in variable space')\n",
    "\n",
    "\n",
    "\n",
    "# and again in the GED space\n",
    "for i in range(2):\n",
    "  axs[1].plot(projA[labels==i,0],projA[labels==i,1],marker[i],label=f'Class {i}')\n",
    "axs[1].set_xlabel('GED axis 1')\n",
    "axs[1].set_ylabel('GED axis 2')\n",
    "axs[1].set_title('Data in GED space')\n",
    "\n",
    "\n",
    "# common settings\n",
    "for i in range(2):\n",
    "  axs[i].axis([-6,6,-6,6])\n",
    "  axs[i].grid()\n",
    "  axs[i].legend()\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('Figure_15_06ab.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "FMchgtL2YKOG"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# prediction (converted to ints)\n",
    "predictedLabel = ( projA[:,0] > 0 )+0\n",
    "\n",
    "print(f'Prediction accuracy: {100*np.mean( predictedLabel==labels )}%')\n",
    "\n",
    "# show the results\n",
    "plt.figure(figsize=(12,5))\n",
    "plt.plot(predictedLabel,'ks',markersize=7,markerfacecolor='w',linewidth=2)\n",
    "plt.plot([N-.5,N-.5],[-.5,1.5],'k--')\n",
    "plt.xlabel('Sample number')\n",
    "plt.ylabel('Predicted class')\n",
    "plt.yticks([0,1],labels=['Class 0','Class 1'])\n",
    "plt.title(f'Accuracy = {100*np.mean(predictedLabel==labels):.2f}%')\n",
    "plt.savefig('Figure_15_06c.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "KvHOAEvqYKRU"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# redraw the jointplot in the GED space (used in Figure 2)\n",
    "ax = sns.jointplot(x=projA[:,0],y=projA[:,1],hue=labels,xlim=[-6,6],ylim=[-6,6])\n",
    "ax.ax_joint.set_xlabel('LDA axis 1')\n",
    "ax.ax_joint.set_ylabel('LDA axis 2')\n",
    "ax.plot_joint(sns.kdeplot)\n",
    "plt.savefig('Figure_15_02b.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "GSW83HkjYKU8"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "5jj6jKPScXiO"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Exercise 7"
   ],
   "metadata": {
    "id": "P0Ff3WDlRSaU"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# not the identity matrix!\n",
    "print(\"V'V:\")\n",
    "print(np.round( evecs.T @ evecs ,3))\n",
    "\n",
    "\n",
    "# yes the identity matrix!\n",
    "print(f\"\\nV'RV:\")\n",
    "print(np.round( evecs.T @ covW @ evecs ,3))"
   ],
   "metadata": {
    "id": "QI3GBeQTd4nK"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "cPHJ5APTRUMa"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Exercise 8"
   ],
   "metadata": {
    "id": "cM6YDugQOBNy"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA\n",
    "\n",
    "ldamodel = LDA(solver='eigen')\n",
    "ldamodel.fit(alldata,labels)\n",
    "\n",
    "\n",
    "# show the results\n",
    "plt.figure(figsize=(12,5))\n",
    "plt.plot(predictedLabel,'ks',markersize=7,markerfacecolor='w',linewidth=2,label='My LDA')\n",
    "plt.plot(ldamodel.predict(alldata),'r+',markersize=10,markerfacecolor='w',linewidth=2,label='sklearn LDA')\n",
    "plt.plot([N-.5,N-.5],[-.5,1.5],'k--')\n",
    "plt.xlabel('Sample number')\n",
    "plt.ylabel('Predicted class')\n",
    "plt.yticks([0,1],labels=['Class 0','Class 1'])\n",
    "plt.ylim([-.5,1.5])\n",
    "plt.legend()\n",
    "plt.title(f'Accuracy = {100*np.mean(ldamodel.predict(alldata)==labels):.2f}%')\n",
    "plt.savefig('Figure_15_07.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "WQ9pagYlODTi"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "VOUEULE5Qd28"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Exercise 9"
   ],
   "metadata": {
    "id": "2isYr4F2QdsQ"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# shrinkage amounts\n",
    "shrinkage = np.linspace(0,1,21)\n",
    "accuracies = np.zeros(len(shrinkage))\n",
    "\n",
    "# loop over shrinkages and compute model accuracy\n",
    "for i,s in enumerate(shrinkage):\n",
    "  \n",
    "  # setup the model\n",
    "  ldamodel = LDA(solver='eigen',shrinkage=s)\n",
    "\n",
    "  tmpacc = []\n",
    "  for _ in range(50):\n",
    "\n",
    "    # randomly split the data into train/test\n",
    "    randorder = np.random.permutation(alldata.shape[0])\n",
    "\n",
    "    # fit the model on the training data\n",
    "    ldamodel.fit(alldata[randorder[:350],:],labels[randorder[:350]])\n",
    "\n",
    "    # grab accuracy\n",
    "    tmpacc.append(100*np.mean(ldamodel.predict(alldata[randorder[350:],:])==labels[randorder[350:]]))\n",
    "\n",
    "  # evaluate model performance on the test data\n",
    "  accuracies[i] = np.mean(tmpacc)\n",
    "\n",
    "\n",
    "# plot!\n",
    "plt.figure(figsize=(8,5))\n",
    "plt.plot(shrinkage,accuracies,'ks-',markersize=10,markerfacecolor='w',linewidth=2)\n",
    "plt.xlabel('Shrinkage amount')\n",
    "plt.ylabel('Prediction accuracy on validation trials')\n",
    "plt.title('Effect of shrinkage on model performance')\n",
    "plt.savefig('Figure_15_08.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "PJd-lNYbODYY"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "CoS2lYfpU3aG"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Exercise 10"
   ],
   "metadata": {
    "id": "sGP19orVryIw"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "from skimage import io,color\n",
    "url = 'https://berggasse19.org/wp-content/uploads/2015/05/stravinsky_picasso_wikipedia.png'\n",
    "\n",
    "# import picture and downsample to 2D\n",
    "strav = io.imread(url) / 255\n",
    "#strav = color.rgb2gray(strav)\n",
    "\n",
    "plt.figure(figsize=(8,8))\n",
    "plt.imshow(strav,cmap='gray')\n",
    "plt.title(f'Matrix size: {strav.shape}, rank: {np.linalg.matrix_rank(strav)}')\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "f7iJ7Z53uGoq"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# SVD\n",
    "U,s,Vt = np.linalg.svd(strav)\n",
    "S = np.zeros_like(strav)\n",
    "np.fill_diagonal(S,s)\n",
    "\n",
    "# show scree plot\n",
    "plt.figure(figsize=(12,4))\n",
    "plt.plot(s[:30],'ks-',markersize=10)\n",
    "plt.xlabel('Component index')\n",
    "plt.ylabel('Singular value')\n",
    "plt.title('Scree plot of Stravinsky picture')\n",
    "plt.grid()\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "Um2zi3w6A4tL"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "fig = plt.figure(figsize=(9,9))\n",
    "gs = GridSpec(3,4,figure=fig)\n",
    "\n",
    "# the image\n",
    "ax1 = fig.add_subplot(gs[0,0])\n",
    "ax1.imshow(strav,cmap='gray')\n",
    "ax1.set_title(f'Matrix size: {strav.shape},\\nrank: {np.linalg.matrix_rank(strav)}')\n",
    "\n",
    "# scree plot\n",
    "ax2 = fig.add_subplot(gs[0,1:])\n",
    "ax2.plot(s[:30],'ks-',markersize=10)\n",
    "ax2.set_xlabel('Component index')\n",
    "ax2.set_ylabel('Singular value')\n",
    "ax2.set_title('Scree plot of Stravinsky picture')\n",
    "ax2.grid()\n",
    "\n",
    "\n",
    "## now show the first N \"layers\" separately\n",
    "numLayers = 4\n",
    "rank1mats = np.zeros((numLayers,strav.shape[0],strav.shape[1]))\n",
    "\n",
    "\n",
    "# the loop\n",
    "for i in range(numLayers):\n",
    "    \n",
    "    # create this layer\n",
    "    rank1mats[i,:,:] = np.outer(U[:,i],Vt[i,:])*s[i]\n",
    "    \n",
    "    # show this layer\n",
    "    ax = fig.add_subplot(gs[1,i])\n",
    "    ax.imshow(rank1mats[i,:,:],cmap='gray')\n",
    "    ax.set_title(f'L {i}')\n",
    "    ax.set_xticks([]), ax.set_yticks([])\n",
    "\n",
    "    # show the cumulative sum of layers\n",
    "    ax = fig.add_subplot(gs[2,i])\n",
    "    ax.imshow(np.sum(rank1mats[:i+1,:,:],axis=0),cmap='gray')\n",
    "    ax.set_title(f'L 0:{i}')\n",
    "    ax.set_xticks([]), ax.set_yticks([])\n",
    "\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('Figure_15_09.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "NwIu6AB7A5Bl"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "ObJ_XiP8AXLR"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Exercise 11"
   ],
   "metadata": {
    "id": "MENNYTJqAUv1"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# Reconstruct based on first k layers\n",
    "\n",
    "# number of components\n",
    "k = 80\n",
    "\n",
    "# reconstruction\n",
    "stravRec = U[:,:k] @ S[:k,:k] @ Vt[:k,:]\n",
    "\n",
    "\n",
    "# show the original, reconstructed, and error\n",
    "_,axs = plt.subplots(1,3,figsize=(15,6))\n",
    "\n",
    "axs[0].imshow(strav,cmap='gray',vmin=.1,vmax=.9)\n",
    "axs[0].set_title('Original')\n",
    "\n",
    "axs[1].imshow(stravRec,cmap='gray',vmin=.1,vmax=.9)\n",
    "axs[1].set_title(f'Reconstructed (k={k}/{len(s)})')\n",
    "\n",
    "axs[2].imshow((strav-stravRec)**2,cmap='gray',vmin=0,vmax=1e-1)\n",
    "axs[2].set_title('Squared errors')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('Figure_15_10.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "8iieJjHUA5Ey"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# compute sizes of the images\n",
    "stravSize  = strav.nbytes / 1024**2\n",
    "stravRSize = stravRec.nbytes / 1024**2\n",
    "\n",
    "# and of the vectors/values\n",
    "uSize = U[:,:k].nbytes / 1024**2\n",
    "sSize = s[:k].nbytes / 1024**2\n",
    "vSize = Vt[:k,:].nbytes / 1024**2\n",
    "\n",
    "\n",
    "# print image sizes\n",
    "print(f'      Original is {stravSize:.2f} mb')\n",
    "print(f'Reconstruction is {stravRSize:.2f} mb')\n",
    "print(f'Recon vectors are {uSize+sSize+vSize:.2f} mb (using k={k} comps.)')\n",
    "\n",
    "print(f'\\nCompression of {100*(uSize+sSize+vSize)/stravSize:.2f}%')"
   ],
   "metadata": {
    "id": "BKLEpbYHIbte"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "sCpNb3q2ef97"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Exercise 12"
   ],
   "metadata": {
    "id": "3nNbhr51efxa"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# range of components\n",
    "k = range(1,len(s)+1)\n",
    "\n",
    "# initialize variable to store results\n",
    "kError = np.zeros(len(k))\n",
    "\n",
    "\n",
    "# the loop\n",
    "for i in range(len(k)):\n",
    "  \n",
    "  # reconstruction\n",
    "  stravRec = U[:,:k[i]] @ S[:k[i],:k[i]] @ Vt[:k[i],:]\n",
    "\n",
    "  # compute and store the error\n",
    "  kError[i] = np.sqrt(np.sum((strav-stravRec)**2))\n",
    "\n",
    "\n",
    "\n",
    "# show the results\n",
    "plt.figure(figsize=(10,7))\n",
    "plt.plot(k,kError,'ks-')\n",
    "# plt.plot(k[:-1],np.diff(kError),'ks-') # uncomment to show derivative (and comment out the previous line)\n",
    "plt.xlabel('Rank of reconstruction')\n",
    "plt.ylabel('Error from original')\n",
    "plt.title('Reconstruction accuracy')\n",
    "plt.savefig('Figure_15_11.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "jJhX1AK7A5H5"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "u3NBQGztWId2"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Exercise 13"
   ],
   "metadata": {
    "id": "73JgL1x7A5LC"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# create a spatial sine wave\n",
    "\n",
    "# sine phases\n",
    "sinefreq = .02   # arbitrary units\n",
    "sinephas = np.pi/6 # rotate\n",
    "\n",
    "# sine wave initializations\n",
    "[x,y] = np.meshgrid(np.linspace(-100,100,strav.shape[1]),\n",
    "                    np.linspace(-100,100,strav.shape[0]))\n",
    "xp    = x*np.cos(sinephas) + y*np.sin(sinephas)\n",
    "\n",
    "\n",
    "# compute sine wave\n",
    "sinimg = np.sin( 2*np.pi*sinefreq*xp)\n",
    "\n",
    "# scale to [0 1]\n",
    "sinimg = (sinimg-np.min(sinimg)) / (np.max(sinimg)-np.min(sinimg))\n",
    "\n",
    "\n",
    "# add to stravinsky picture and re-scale (using two lines)\n",
    "stravNoise = strav + sinimg\n",
    "stravNoise = stravNoise-np.min(stravNoise)\n",
    "stravNoise = stravNoise/np.max(stravNoise)\n",
    "\n",
    "# let's see it!\n",
    "_,axs = plt.subplots(1,3,figsize=(10,7))\n",
    "axs[0].imshow(strav,cmap='gray')\n",
    "axs[0].set_title('Original picture')\n",
    "\n",
    "axs[1].imshow(sinimg,cmap='gray')\n",
    "axs[1].set_title('Noise image')\n",
    "\n",
    "axs[2].imshow(stravNoise,cmap='gray')\n",
    "axs[2].set_title('Contaminated picture')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('Figure_15_12.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "0BQ7UkH3wnga"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# SVD\n",
    "Un,sn,Vtn = np.linalg.svd(stravNoise)\n",
    "Sn = np.zeros_like(stravNoise)\n",
    "np.fill_diagonal(Sn,sn)\n",
    "\n",
    "# show scree plot\n",
    "plt.figure(figsize=(12,4))\n",
    "plt.plot(sn[:30],'ks-',markersize=10)\n",
    "plt.xlabel('Component index')\n",
    "plt.ylabel('Singular value')\n",
    "plt.title('Scree plot of Noisy Stravinsky picture')\n",
    "plt.grid()\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "2zbF5Erswnjw"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "fig = plt.figure(figsize=(9,9))\n",
    "gs = GridSpec(3,4,figure=fig)\n",
    "\n",
    "# the image\n",
    "ax1 = fig.add_subplot(gs[0,0])\n",
    "ax1.imshow(stravNoise,cmap='gray')\n",
    "ax1.set_title(f'Matrix size: {strav.shape},\\nrank: {np.linalg.matrix_rank(stravNoise)}')\n",
    "\n",
    "# scree plot\n",
    "ax2 = fig.add_subplot(gs[0,1:])\n",
    "ax2.plot(sn[:30],'ks-',markersize=10)\n",
    "ax2.set_xlabel('Component index')\n",
    "ax2.set_ylabel('Singular value')\n",
    "ax2.set_title('Scree plot of noisy Stravinsky picture')\n",
    "ax2.grid()\n",
    "\n",
    "\n",
    "## now show the first N \"layers\" separately\n",
    "numLayers = 4\n",
    "rank1mats = np.zeros((numLayers,strav.shape[0],strav.shape[1]))\n",
    "\n",
    "\n",
    "# the loop\n",
    "for i in range(numLayers):\n",
    "    \n",
    "    # create this layer\n",
    "    rank1mats[i,:,:] = np.outer(Un[:,i],Vtn[i,:])*sn[i]\n",
    "    \n",
    "    # show this layer\n",
    "    ax = fig.add_subplot(gs[1,i])\n",
    "    ax.imshow(rank1mats[i,:,:],cmap='gray')\n",
    "    ax.set_title(f'L {i}')\n",
    "    ax.set_xticks([]), ax.set_yticks([])\n",
    "\n",
    "    # show the cumulative sum of layers\n",
    "    ax = fig.add_subplot(gs[2,i])\n",
    "    ax.imshow(np.sum(rank1mats[:i+1,:,:],axis=0),cmap='gray')\n",
    "    ax.set_title(f'L 0:{i}')\n",
    "    ax.set_xticks([]), ax.set_yticks([])\n",
    "\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('Figure_15_13.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "lCWmIFncM3lD"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "iqd-RCpSravh"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Exercise 14"
   ],
   "metadata": {
    "id": "PB3flbY-ralT"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# Reconstruct without specified layers\n",
    "\n",
    "# noise components\n",
    "noiseComps = np.array([1,2])\n",
    "\n",
    "# reconstruction of the noise\n",
    "stravRecNoise = Un[:,noiseComps] @ Sn[noiseComps,:][:,noiseComps] @ Vtn[noiseComps,:]\n",
    "\n",
    "\n",
    "# reconstruction of the image with noise projected out\n",
    "noNoiseCompsU = np.full(Un.shape[0],True)\n",
    "noNoiseCompsU[noiseComps] = False\n",
    "\n",
    "noNoiseCompsV = np.full(Vtn.shape[0],True)\n",
    "noNoiseCompsV[noiseComps] = False\n",
    "\n",
    "# here's the image without the noise components\n",
    "stravRecNoNoise = Un[:,noNoiseCompsU] @ Sn[noNoiseCompsU,:][:,noNoiseCompsV] @ Vtn[noNoiseCompsV,:]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# show the original, reconstructed, and error\n",
    "_,axs = plt.subplots(1,3,figsize=(15,6))\n",
    "\n",
    "axs[0].imshow(stravNoise,cmap='gray')\n",
    "axs[0].set_title('Noisy image')\n",
    "\n",
    "axs[1].imshow(stravRecNoise,cmap='gray',vmin=-.5,vmax=.5)\n",
    "axs[1].set_title(f'Only noise (comps {noiseComps})')\n",
    "\n",
    "axs[2].imshow(stravRecNoNoise,cmap='gray',vmin=.1,vmax=.9)\n",
    "axs[2].set_title('Noise projected out')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('Figure_15_14.png',dpi=300)\n",
    "plt.show()"
   ],
   "metadata": {
    "id": "-fezsrBewnqP"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# histogram of noise reconstruction\n",
    "plt.hist(stravRecNoise.flatten(),100);"
   ],
   "metadata": {
    "id": "9lU_sEYRwntk"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    ""
   ],
   "metadata": {
    "id": "yeM4MtNGx4Xh"
   },
   "execution_count": null,
   "outputs": []
  }
 ]
}