developing.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "Python 3.8.3 64-bit (conda)",
   "display_name": "Python 3.8.3 64-bit (conda)",
   "metadata": {
    "interpreter": {
     "hash": "d89d7567fcc2ae56ec5f438db992621a22fcaec829c985c72867f61b5c1db2c1"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "y1 = torch.tensor([[0.5, 0.2, 0.05, 0.1, 0.05, 0.1],[0.5, 0.2, 0.05, 0.1, 0.05, 0.1]], requires_grad=True)\n",
    "y2 = torch.tensor([[0.5, 0.2, 0.1, 0.05, 0.1, 0.05],[0.5, 0.2, 0.1, 0.05, 0.1, 0.05]], requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MSEEntropyLoss(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MSEEntropyLoss, self).__init__()\n",
    "        self.MSE = nn.MSELoss()\n",
    "    def forward(self, p_target: torch.Tensor, p_estimate: torch.Tensor):\n",
    "        assert p_target.shape == p_estimate.shape\n",
    "        entropy_target = torch.sum(-p_target.mul(p_target.log()), dim=1)\n",
    "        entropy_estimate = torch.sum(-p_estimate.mul(p_estimate.log()), dim=1)\n",
    "        entropy_diff = self.MSE(entropy_estimate, entropy_target)\n",
    "        return entropy_diff.mean()\n",
    "MSEEntropy = MSEEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(tensor([[0.5000, 0.2000, 0.0500, 0.1000, 0.0500, 0.1000],\n         [0.5000, 0.2000, 0.0500, 0.1000, 0.0500, 0.1000]]),\n tensor([[-0.6931, -1.6094, -2.9957, -2.3026, -2.9957, -2.3026],\n         [-0.6931, -1.6094, -2.9957, -2.3026, -2.9957, -2.3026]]),\n tensor([[-0.3466, -0.3219, -0.1498, -0.2303, -0.1498, -0.2303],\n         [-0.3466, -0.3219, -0.1498, -0.2303, -0.1498, -0.2303]]),\n tensor([-1.4286, -1.4286]))"
     },
     "metadata": {},
     "execution_count": 34
    }
   ],
   "source": [
    "y1, y1.log(), y1*y1.log(),  torch.sum(y1*y1.log(), dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = MSEEntropy(y1, y2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(tensor([[ 3.6580e-08, -7.2651e-08, -2.3791e-07, -1.5528e-07, -2.3791e-07,\n          -1.5528e-07],\n         [ 3.6580e-08, -7.2651e-08, -2.3791e-07, -1.5528e-07, -2.3791e-07,\n          -1.5528e-07]]),\n tensor([[-3.6580e-08,  7.2651e-08,  1.5528e-07,  2.3791e-07,  1.5528e-07,\n           2.3791e-07],\n         [-3.6580e-08,  7.2651e-08,  1.5528e-07,  2.3791e-07,  1.5528e-07,\n           2.3791e-07]]))"
     },
     "metadata": {},
     "execution_count": 40
    }
   ],
   "source": [
    "y1.grad , y2.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(tensor([[0.5000, 0.2000, 0.0500, 0.1000, 0.0500, 0.1000],\n         [0.5000, 0.2000, 0.0500, 0.1000, 0.0500, 0.1000]], requires_grad=True),\n tensor([[0.5000, 0.2000, 0.1000, 0.0500, 0.1000, 0.0500],\n         [0.5000, 0.2000, 0.1000, 0.0500, 0.1000, 0.0500]], requires_grad=True))"
     },
     "metadata": {},
     "execution_count": 41
    }
   ],
   "source": [
    "y1, y2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}