voting_scheme.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import math\n",
    "import statistics\n",
    "import pandas as pd\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as mpatches\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load output test probabilitites and true labels for each model\n",
    "\n",
    "For every model there are 2 files generated in numpy binary format:\n",
    "\n",
    "  **probabilities.npy** - numpy array of size **TEST POINT CLOUDS CLOUD x CLASSES COUNT x MODELS COUNT** with output classification probability for each of 10 individually trained models\n",
    "\n",
    "  **true_labels.npy** - numpy array of size **TEST POINT CLOUDS CLOUD** with true labels data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Base models\n",
    "BASE_MODELS = ['pointnet', 'pointnet++', 'so-net', 'kcnet', 'deepsets', 'dgcnn', 'pointcnn']\n",
    "BASE_MODELS_ALIASES = ['PointNet', 'PointNet++', 'SO-Net', 'KCNet', 'DeepSets', 'DGCNN', 'PointCNN']\n",
    "\n",
    "CLASS_NAMES = ['airplane', 'bathtub', 'bed', 'bench', 'bookshelf', 'bottle', 'bowl',\n",
    "               'car', 'chair', 'cone', 'cup', 'curtain', 'desk', 'door', 'dresser',\n",
    "               'flower_pot', 'glass_box', 'guitar', 'keyboard', 'lamp', 'laptop',\n",
    "               'mantel', 'monitor', 'night_stand', 'person', 'piano', 'plant', 'radio',\n",
    "               'range_hood', 'sink', 'sofa', 'stairs', 'stool', 'table', 'tent', 'toilet',\n",
    "               'tv_stand', 'vase', 'wardrobe', 'xbox']\n",
    "CLASS_TEST_INSTANCES = [100, 50, 100, 20, 100, 100, 20, 100, 100, 20, 20, 20, 86, 20,\n",
    "                        86, 20, 100, 100, 20, 20, 20, 100, 100, 86, 20, 100, 100, 20,\n",
    "                        100, 20, 100, 20, 20, 100, 20, 100, 100, 100, 20, 20]\n",
    "CLASSES_COUNT = 40\n",
    "\n",
    "probabilities = {}\n",
    "true_labels = {}\n",
    "for model in list(set(BASE_MODELS) - {'deepsets'}):\n",
    "    for dir_name in ['models_reordered']:\n",
    "        if os.path.exists(os.path.join(dir_name, model)):\n",
    "            break\n",
    "    prob_filepath = os.path.join(os.path.join(dir_name, model), 'probabilities.npy')\n",
    "    label_filepath = os.path.join(os.path.join(dir_name, model), 'true_labels.npy')\n",
    "    probabilities[model] = np.load(prob_filepath)\n",
    "    if dir_name != 'models_reordered':\n",
    "        probabilities[model] = np.transpose(probabilities[model], axes=(2, 0, 1))\n",
    "\n",
    "    # probabilities[model] =  softmax(probabilities[model])\n",
    "    idx = np.argmax(probabilities[model], axis=-1)\n",
    "    hardvote = np.squeeze(np.eye(40)[idx.reshape(-1)]).reshape(probabilities[model].shape)\n",
    "    e_x = np.exp(probabilities[model])\n",
    "    softvote = e_x /np.expand_dims(np.sum(e_x, axis=-1), axis=-1)\n",
    "    probabilities[model] = {'activations': probabilities[model]/np.std(probabilities[model]),\n",
    "                           'hardvote': hardvote, 'softvote': softvote}\n",
    "    true_labels[model] = np.load(label_filepath)\n",
    "    \n",
    "    for class_idx in range(CLASSES_COUNT):\n",
    "        error_info = (\"Test cloud instances for model \" + model + \" and class \"\n",
    "                      + str(class_idx) + \" differs! Should be \" + str(CLASS_TEST_INSTANCES[class_idx])\n",
    "                      + \" but got \" + str(np.sum(true_labels[model] == class_idx)))\n",
    "        assert np.sum(true_labels[model] == class_idx) == CLASS_TEST_INSTANCES[class_idx] or \\\n",
    "          (model=='deepsets' and class_idx==12 and np.sum(true_labels[model] == class_idx) == 85) or \\\n",
    "          (model=='deepsets' and class_idx==39 and np.sum(true_labels[model] == class_idx) == 19) or \\\n",
    "          (model=='deepsets_ep100' and class_idx==12 and np.sum(true_labels[model] == class_idx) == 85) or \\\n",
    "          (model=='deepsets_ep100' and class_idx==39 and np.sum(true_labels[model] == class_idx) == 19), \\\n",
    "          error_info\n",
    "                \n",
    "#     assert (probabilities[model].shape[0] == ENSEMBLE_MODELS_NUMBER)\n",
    "    assert (probabilities[model]['activations'].shape[1] == 2468 or \\\n",
    "            (model=='deepsets' and probabilities[model]['activations'].shape[1] == 2466) or \\\n",
    "           (model=='deepsets_ep100' and probabilities[model]['activations'].shape[1] == 2466))\n",
    "    assert (probabilities[model]['activations'].shape[2] == CLASSES_COUNT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data storage\n",
    "\n",
    "Probabilites variable is built as follows:\n",
    "\n",
    "{ architecture: { voting_scheme :  np.array of shape (I, N, C) }}\n",
    "\n",
    "where:\n",
    " - I : individually trained model index - 10 training sessions were performed for the purpose of the article\n",
    " - N : point cloud index - 2468 point clouds of the ModelNet40 dataset\n",
    " - C : output score for each class - 40 classe of the ModelNet40 dataset\n",
    " \n",
    "To get output scores (activations) of pointnet architecture of the 3rd trained model for 1187th point cloud one could type:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-3.02531755 -1.37491622 -1.50965321 -0.88119054  0.1614824   0.10937071\n",
      " -2.19930438 -2.31120737 -1.52770762 -2.0513665  -0.85198568 -0.06116966\n",
      " -2.13772444 -0.12316392  0.37912571 -0.68710185 -0.91850996 -1.44277731\n",
      " -0.96027636 -1.26352561 -2.2914994  -0.85541151 -1.32968591 -0.57514311\n",
      " -1.21187871 -1.57663164 -1.92808107 -0.19898758 -1.6192764  -0.88746693\n",
      " -1.97757792 -1.52654201 -0.99189835 -1.0141773  -1.75180487 -1.67449783\n",
      " -0.45883901 -0.24417091  0.44780099 -0.69122378]\n"
     ]
    }
   ],
   "source": [
    "model_no = 3\n",
    "point_cloud_no = 1187\n",
    "\n",
    "print (probabilities['pointnet']['activations'][model_no][point_cloud_no])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y = 19 \ty_hat = 38\n"
     ]
    }
   ],
   "source": [
    "# Best score for above point cloud is obtained for:\n",
    "y_hat = np.argmax(probabilities['pointnet']['activations'][model_no][point_cloud_no])\n",
    "\n",
    "# True label for the above point cloud\n",
    "y = true_labels['pointnet'][point_cloud_no]\n",
    "\n",
    "# Info\n",
    "print ('y =', y, '\\ty_hat =', y_hat)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}