embedding_reference_feature.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import cv2\n",
    "import sys\n",
    "import itertools\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from configs.config import Config\n",
    "from libs.networks.network_factory import get_network_byname\n",
    "from libs import build_rpn, build_fast_rcnn, build_fpn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"3\"\n",
    "id_to_class = {1:'ascus', 2:'asch', \n",
    "  3:'lsil', 4:'hsil', 5:'scc',  6:'agc', 7:'trichomonas', \n",
    " 8:'candida',  9:'flora', 10:'herps', 11:'actinomyces'}\n",
    "# set file of showing image and labels \n",
    "path_for_metadata = os.path.join(os.getcwd(), \"labels.tsv\")\n",
    "path_sprite_image = os.path.join(os.getcwd(), \"sprite_image.bmp\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### make sprite image and metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_sprite_image(images, width, height, number_padding=4): \n",
    "    \"\"\"\n",
    "    images: the list images for make up sprite image\n",
    "    width: the number images along with width\n",
    "    height: the number image along with height\n",
    "    number_padding: the pixels padding along with images\n",
    "    Returns a sprite image consisting of images passed as argument. \n",
    "    Images should be count x width x height\"\"\"\n",
    "    if isinstance(images, list):\n",
    "        images = np.array(images)\n",
    "    img_h = images.shape[1] \n",
    "    img_w = images.shape[2]\n",
    "    assert width * height == images.shape[0]\n",
    "    spriteimage = np.ones(((img_h +2 * number_padding) * height,\n",
    "                           (img_w +2 * number_padding) * width, 3)) * 255\n",
    "    \n",
    "    for i in range(height): \n",
    "        for j in range(width):\n",
    "            this_filter = i * width + j\n",
    "            if this_filter < images.shape[0]: \n",
    "                this_img = images[this_filter] \n",
    "                spriteimage[i * (img_h+2 * number_padding) + number_padding :\n",
    "                            (i + 1) * (img_h+2*number_padding) - number_padding,\n",
    "                            j * (img_w+2 * number_padding) + number_padding :\n",
    "                            (j + 1) * (img_w+2*number_padding) - number_padding] = this_img\n",
    "    return spriteimage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### load reference image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_labels = []\n",
    "sample_images = []\n",
    "sample_images_name = []\n",
    "train_images = []\n",
    "\n",
    "for l in os.listdir(\"./images/\"):\n",
    "    label_path = os.path.join(\"./images/\", l)\n",
    "    if os.path.isdir(label_path):\n",
    "        for i in os.listdir(label_path):\n",
    "            item_image = cv2.imread(os.path.join(label_path, i))\n",
    "            train_images.append(cv2.resize(item_image,(224,224)))\n",
    "            item_image = cv2.resize(item_image,(28,28))\n",
    "            sample_images_name.append(int(i.split(\".\")[0]))\n",
    "            sample_images.append(item_image)\n",
    "            sample_labels.append(id_to_class[int(l)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### shuffle the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_images = np.array(sample_images)\n",
    "sample_labels = np.array(sample_labels)\n",
    "sample_images_name = np.array(sample_images_name)\n",
    "train_images = np.array(train_images)\n",
    "# shuffle the data\n",
    "shuffle_index = np.random.permutation(sample_images.shape[0])\n",
    "train_images = train_images[shuffle_index,:,:,:]\n",
    "sample_images = sample_images[shuffle_index,:,:,:]\n",
    "sample_labels = sample_labels[shuffle_index]\n",
    "sample_images_name = sample_images_name[shuffle_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1130"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_images_name[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "sprite_image = create_sprite_image(sample_images[:39*40,:,:,:], 39, 40)\n",
    "cv2.imwrite(path_sprite_image, sprite_image)\n",
    "\n",
    "# write the corresponding labels into metadata\n",
    "with open(path_for_metadata,'w') as f: \n",
    "    f.write(\"Index\\tLabel\\n\") \n",
    "    for index,label in zip(sample_images_name[:39*40], sample_labels[:39*40]): \n",
    "        f.write(\"%d\\t%s\\n\" % (index,label))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Embedding the reference images "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### setup embedding model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_fn(features,\n",
    "             mode,\n",
    "             params,\n",
    "             config):\n",
    "    net_config = params[\"net_config\"]\n",
    "    IS_TRAINING = False\n",
    "    origin_image_batch = tf.cast(features, tf.float32)\n",
    "    image_batch = origin_image_batch - net_config.PIXEL_MEANS\n",
    "    # there is is_training means that bn is training, so it is important!\n",
    "    _, share_net = get_network_byname(inputs=image_batch,\n",
    "                                      config=net_config,\n",
    "                                      is_training=IS_TRAINING,\n",
    "                                      reuse=tf.AUTO_REUSE)\n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        predicts = {\"embedding_feature\": share_net[\"C5\"]}\n",
    "\n",
    "    return tf.estimator.EstimatorSpec(mode, predictions=predicts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_model_dir': './logs', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': gpu_options {\n",
      "  allow_growth: true\n",
      "}\n",
      "allow_soft_placement: true\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f065248bf98>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n"
     ]
    }
   ],
   "source": [
    "net_config = Config()\n",
    "session_config = tf.ConfigProto()\n",
    "session_config.gpu_options.allow_growth = True\n",
    "session_config.allow_soft_placement = True\n",
    "estimator_config = tf.estimator.RunConfig(model_dir=net_config.MODLE_DIR,\n",
    "                                          session_config=session_config)\n",
    "\n",
    "my_estimator = tf.estimator.Estimator(model_fn,\n",
    "                                      params={\"net_config\": net_config}, \n",
    "                                      config=estimator_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_reference_batch(images):\n",
    "    reference_data = tf.data.Dataset.from_tensor_slices(images)\n",
    "    return reference_data.batch(32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "single_record = my_estimator.predict(input_fn=lambda:build_reference_batch(train_images),\n",
    "                                     yield_single_examples=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from ./logs/model.ckpt-224388\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    }
   ],
   "source": [
    "total_pyramid_feature = []\n",
    "while True:\n",
    "    try:\n",
    "        pyramid_feature = next(single_record)[\"embedding_feature\"]\n",
    "        total_pyramid_feature.append(pyramid_feature)\n",
    "    except:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1563, 7, 7, 2048)\n",
      "(1560, 100352)\n"
     ]
    }
   ],
   "source": [
    "all_pyramid_feature = np.concatenate(total_pyramid_feature, axis=0)\n",
    "print(np.shape(all_pyramid_feature))\n",
    "# Arrangement the reference image feature\n",
    "embedding_input = all_pyramid_feature[:39*40,:, :, :]\n",
    "embedding_input = np.reshape(embedding_input, (np.shape(embedding_input)[0], -1))\n",
    "embedding_size = np.shape(embedding_input)\n",
    "print(embedding_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### setup the embedding function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'embedding/model.ckpt-1'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding = tf.Variable(tf.zeros(embedding_size), name=\"test_embedding\")\n",
    "assignment = embedding.assign(embedding_input)\n",
    "\n",
    "writer = tf.summary.FileWriter(\"embedding/\")\n",
    "config = tf.contrib.tensorboard.plugins.projector.ProjectorConfig()\n",
    "embedding_config = config.embeddings.add()\n",
    "embedding_config.tensor_name = embedding.name\n",
    "embedding_config.sprite.image_path = path_sprite_image\n",
    "embedding_config.metadata_path = path_for_metadata\n",
    "# Specify the width and height of a single thumbnail.\n",
    "embedding_config.sprite.single_image_dim.extend([36, 36])\n",
    "tf.contrib.tensorboard.plugins.projector.visualize_embeddings(writer, config)\n",
    "sess = tf.InteractiveSession()\n",
    "saver = tf.train.Saver()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "sess.run(assignment)\n",
    "saver.save(sess, os.path.join(\"embedding\",\"model.ckpt\"), 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K-means cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "class_list = [\"ascu\", \"asch\", \"lsil\",\"hsil\", \"scc\", \"agc\", \"trich\", \"cand\", \"flora\", \"herps\", \"actin\"]\n",
    "number_list = [3, 4, 2, 4, 2, 3, 1, 2, 2, 4, 1]\n",
    "sample_labels = sample_labels[:39*40]\n",
    "sample_images_name = sample_images_name[:39*40]\n",
    "\n",
    "class_to_id = {}\n",
    "for key in id_to_class:\n",
    "    class_to_id[id_to_class[key]] = key\n",
    "sample_labels_id = np.zeros_like(sample_labels)\n",
    "for i in range(np.shape(sample_labels)[0]):\n",
    "    sample_labels_id[i] = class_to_id[sample_labels[i]]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[845, 1075, 257]\n",
      "[278, 910, 93]\n",
      "[752, 974, 617]\n",
      "[293, 1629, 738]\n",
      "[761, 659, 1049]\n",
      "[1059, 552, 112]\n",
      "[998, 587, 537]\n",
      "[1207, 1002, 1346]\n",
      "[568, 1100, 1448]\n",
      "[884, 1080, 1213]\n",
      "[1248, 730, 328]\n"
     ]
    }
   ],
   "source": [
    "for i in range(1, 12):\n",
    "    tmp_number = number_list[i-1]\n",
    "    tmp_features = embedding_input[sample_labels_id==str(i)]\n",
    "    tmp_image_name = sample_images_name[sample_labels_id==str(i)]\n",
    "    kmeans = KMeans(n_clusters=tmp_number, random_state=0).fit(tmp_features)\n",
    "    tmp_expand_features = np.expand_dims(tmp_features, 1)\n",
    "    center_expand_features = np.expand_dims(kmeans.cluster_centers_, 0)\n",
    "    tmp_diff = np.sum(np.abs(tmp_expand_features-center_expand_features), 2)\n",
    "    origin_shape = tmp_diff.shape\n",
    "    tmp_diff = np.reshape(tmp_diff, (-1))\n",
    "    sort_index = np.argsort(tmp_diff)\n",
    "    concat_name_list = []\n",
    "    for j in range(3):\n",
    "        concat_name_list.append(tmp_image_name[sort_index[j]//tmp_number + 1])\n",
    "    print(concat_name_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}