speech-to-text/wav2vec-pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import librosa\n",
    "# import tensorflow as tf\n",
    "import glob\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# follow hyperparameters from here, https://github.com/pytorch/fairseq/tree/master/examples/wav2vec\n",
    "\n",
    "features = [(512, 10, 5), (512, 8, 4), (512, 4, 2), (512, 4, 2), (512, 4, 2), (512, 1, 1), (512, 1, 1)]\n",
    "aggs = [(512, 2, 1), (512, 3, 1), (512, 4, 1), (512, 5, 1), (512, 6, 1), (512, 7, 1), (512, 8, 1), (512, 9, 1), \n",
    "        (512, 10, 1), (512, 11, 1), (512, 12, 1), (512, 13, 1)]\n",
    "num_negatives = 10\n",
    "prediction_steps = 12\n",
    "learning_rate = 1e-5\n",
    "min_learning_rate = 1e-9\n",
    "max_learning_rate = 0.005\n",
    "learning_scheduler = 'cosine'\n",
    "max_update = 400000\n",
    "residual_scale = 0.5\n",
    "log_compression = True\n",
    "warmup_updates = 50\n",
    "warmup_init_lr = 1e-07\n",
    "batch_size = 32\n",
    "epoch = 10\n",
    "total_steps = batch_size * epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torch.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 1.62434536, -0.61175641, -0.52817175, -1.07296862,\n",
       "          0.86540763, -2.3015387 ,  1.74481176],\n",
       "        [-0.7612069 ,  0.3190391 , -0.24937038,  1.46210794,\n",
       "         -2.06014071, -0.3224172 , -0.38405435],\n",
       "        [ 1.13376944, -1.09989127, -0.17242821, -0.87785842,\n",
       "          0.04221375,  0.58281521, -1.10061918],\n",
       "        [ 1.14472371,  0.90159072,  0.50249434,  0.90085595,\n",
       "         -0.68372786, -0.12289023, -0.93576943],\n",
       "        [-0.26788808,  0.53035547, -0.69166075, -0.39675353,\n",
       "         -0.6871727 , -0.84520564, -0.67124613],\n",
       "        [-0.0126646 , -1.11731035,  0.2344157 ,  1.65980218,\n",
       "          0.74204416, -0.19183555, -0.88762896],\n",
       "        [-0.74715829,  1.6924546 ,  0.05080775, -0.63699565,\n",
       "          0.19091548,  2.10025514,  0.12015895],\n",
       "        [ 0.61720311,  0.30017032, -0.35224985, -1.1425182 ,\n",
       "         -0.34934272, -0.20889423,  0.58662319],\n",
       "        [ 0.83898341,  0.93110208,  0.28558733,  0.88514116,\n",
       "         -0.75439794,  1.25286816,  0.51292982],\n",
       "        [-0.29809284,  0.48851815, -0.07557171,  1.13162939,\n",
       "          1.51981682,  2.18557541, -1.39649634]],\n",
       "\n",
       "       [[-1.44411381, -0.50446586,  0.16003707,  0.87616892,\n",
       "          0.31563495, -2.02220122, -0.30620401],\n",
       "        [ 0.82797464,  0.23009474,  0.76201118, -0.22232814,\n",
       "         -0.20075807,  0.18656139,  0.41005165],\n",
       "        [ 0.19829972,  0.11900865, -0.67066229,  0.37756379,\n",
       "          0.12182127,  1.12948391,  1.19891788],\n",
       "        [ 0.18515642, -0.37528495, -0.63873041,  0.42349435,\n",
       "          0.07734007, -0.34385368,  0.04359686],\n",
       "        [-0.62000084,  0.69803203, -0.44712856,  1.2245077 ,\n",
       "          0.40349164,  0.59357852, -1.09491185],\n",
       "        [ 0.16938243,  0.74055645, -0.9537006 , -0.26621851,\n",
       "          0.03261455, -1.37311732,  0.31515939],\n",
       "        [ 0.84616065, -0.85951594,  0.35054598, -1.31228341,\n",
       "         -0.03869551, -1.61577235,  1.12141771],\n",
       "        [ 0.40890054, -0.02461696, -0.77516162,  1.27375593,\n",
       "          1.96710175, -1.85798186,  1.23616403],\n",
       "        [ 1.62765075,  0.3380117 , -1.19926803,  0.86334532,\n",
       "         -0.1809203 , -0.60392063, -1.23005814],\n",
       "        [ 0.5505375 ,  0.79280687, -0.62353073,  0.52057634,\n",
       "         -1.14434139,  0.80186103,  0.0465673 ]]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "x = np.random.normal(size = (2, 10, 7))\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 10, 7])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.from_numpy(x)\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 10, 7)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bsz, fsz, tsz = x.shape\n",
    "bsz, fsz, tsz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10, 14])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = x.transpose(0, 1)\n",
    "y = y.contiguous().view(fsz, -1)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "high = tsz\n",
    "n_negatives = 10\n",
    "# neg_idxs = torch.randint(low=0, high=high, size=(bsz, n_negatives * tsz))\n",
    "# neg_idxs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "neg_idxs = torch.from_numpy(np.array([[\n",
    "         1, 2, 3, 1, 4, 0, 5, 6, 1, 2, 0, 4, 2, 1, 0, 5, 4, 5, 4, 6, 6, 4, 1, 6,\n",
    "         6, 3, 4, 4, 5, 0, 1, 5, 4, 4, 1, 1, 0, 2, 0, 6, 2, 6, 3, 4, 5, 6, 2, 4,\n",
    "         0, 2, 1, 2, 6, 4, 2, 4, 0, 2, 4, 2, 1, 0, 4, 6, 6, 4, 4, 2, 3, 4],\n",
    "        [4, 0, 3, 4, 2, 4, 4, 1, 0, 6, 3, 1, 5, 6, 4, 3, 6, 4, 0, 5, 1, 0, 4, 2,\n",
    "         2, 0, 4, 1, 4, 3, 2, 2, 0, 4, 2, 3, 4, 6, 6, 2, 4, 0, 3, 1, 6, 2, 4, 5,\n",
    "         1, 3, 1, 3, 3, 1, 3, 0, 3, 6, 0, 5, 2, 4, 5, 6, 0, 1, 2, 3, 6, 3]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1,  2,  3,  1,  4,  0,  5,  6,  1,  2,  0,  4,  2,  1,  0,  5,  4,  5,\n",
       "          4,  6,  6,  4,  1,  6,  6,  3,  4,  4,  5,  0,  1,  5,  4,  4,  1,  1,\n",
       "          0,  2,  0,  6,  2,  6,  3,  4,  5,  6,  2,  4,  0,  2,  1,  2,  6,  4,\n",
       "          2,  4,  0,  2,  4,  2,  1,  0,  4,  6,  6,  4,  4,  2,  3,  4],\n",
       "        [11,  7, 10, 11,  9, 11, 11,  8,  7, 13, 10,  8, 12, 13, 11, 10, 13, 11,\n",
       "          7, 12,  8,  7, 11,  9,  9,  7, 11,  8, 11, 10,  9,  9,  7, 11,  9, 10,\n",
       "         11, 13, 13,  9, 11,  7, 10,  8, 13,  9, 11, 12,  8, 10,  8, 10, 10,  8,\n",
       "         10,  7, 10, 13,  7, 12,  9, 11, 12, 13,  7,  8,  9, 10, 13, 10]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(1, bsz):\n",
    "    neg_idxs[i] += i * high\n",
    "    \n",
    "neg_idxs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10, 140])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "negs = y[..., neg_idxs.view(-1)]\n",
    "negs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.6118, -0.5282, -1.0730,  ...,  0.8762, -0.3062,  0.8762],\n",
       "        [ 0.3190, -0.2494,  1.4621,  ..., -0.2223,  0.4101, -0.2223],\n",
       "        [-1.0999, -0.1724, -0.8779,  ...,  0.3776,  1.1989,  0.3776],\n",
       "        ...,\n",
       "        [ 0.3002, -0.3522, -1.1425,  ...,  1.2738,  1.2362,  1.2738],\n",
       "        [ 0.9311,  0.2856,  0.8851,  ...,  0.8633, -1.2301,  0.8633],\n",
       "        [ 0.4885, -0.0756,  1.1316,  ...,  0.5206,  0.0466,  0.5206]],\n",
       "       dtype=torch.float64)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "negs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10, 2, 10, 7])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "negs = negs.view(fsz, bsz, n_negatives, tsz).permute(2, 1, 0, 3)\n",
    "negs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.6118, -0.5282, -1.0730, -0.6118,  0.8654,  1.6243, -2.3015],\n",
       "         [ 0.3190, -0.2494,  1.4621,  0.3190, -2.0601, -0.7612, -0.3224],\n",
       "         [-1.0999, -0.1724, -0.8779, -1.0999,  0.0422,  1.1338,  0.5828],\n",
       "         [ 0.9016,  0.5025,  0.9009,  0.9016, -0.6837,  1.1447, -0.1229],\n",
       "         [ 0.5304, -0.6917, -0.3968,  0.5304, -0.6872, -0.2679, -0.8452],\n",
       "         [-1.1173,  0.2344,  1.6598, -1.1173,  0.7420, -0.0127, -0.1918],\n",
       "         [ 1.6925,  0.0508, -0.6370,  1.6925,  0.1909, -0.7472,  2.1003],\n",
       "         [ 0.3002, -0.3522, -1.1425,  0.3002, -0.3493,  0.6172, -0.2089],\n",
       "         [ 0.9311,  0.2856,  0.8851,  0.9311, -0.7544,  0.8390,  1.2529],\n",
       "         [ 0.4885, -0.0756,  1.1316,  0.4885,  1.5198, -0.2981,  2.1856]],\n",
       "\n",
       "        [[ 0.3156, -1.4441,  0.8762,  0.3156,  0.1600,  0.3156,  0.3156],\n",
       "         [-0.2008,  0.8280, -0.2223, -0.2008,  0.7620, -0.2008, -0.2008],\n",
       "         [ 0.1218,  0.1983,  0.3776,  0.1218, -0.6707,  0.1218,  0.1218],\n",
       "         [ 0.0773,  0.1852,  0.4235,  0.0773, -0.6387,  0.0773,  0.0773],\n",
       "         [ 0.4035, -0.6200,  1.2245,  0.4035, -0.4471,  0.4035,  0.4035],\n",
       "         [ 0.0326,  0.1694, -0.2662,  0.0326, -0.9537,  0.0326,  0.0326],\n",
       "         [-0.0387,  0.8462, -1.3123, -0.0387,  0.3505, -0.0387, -0.0387],\n",
       "         [ 1.9671,  0.4089,  1.2738,  1.9671, -0.7752,  1.9671,  1.9671],\n",
       "         [-0.1809,  1.6277,  0.8633, -0.1809, -1.1993, -0.1809, -0.1809],\n",
       "         [-1.1443,  0.5505,  0.5206, -1.1443, -0.6235, -1.1443, -1.1443]]],\n",
       "       dtype=torch.float64)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "negs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 2, 10, 7]) torch.Size([10, 2, 10, 7])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([11, 2, 10, 7])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = x[:].unsqueeze(0)\n",
    "print(y.shape, negs.shape)\n",
    "targets = torch.cat([y, negs], dim=0)\n",
    "targets.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([11, 2, 10, 7, 12])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "project_to_steps = nn.ConvTranspose2d(10, 10, (1, 12))\n",
    "s = project_to_steps(x.unsqueeze(-1).float()).unsqueeze(0).expand(targets.size(0), -1, -1, -1, -1)\n",
    "s.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('convtranspose.pkl', 'wb') as fopen:\n",
    "    pickle.dump(s.detach().numpy().tolist(), fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "\n",
    "jin = 0\n",
    "rin = 0\n",
    "for _, k, stride in features:\n",
    "    if rin == 0:\n",
    "        rin = k\n",
    "    rin = rin + (k - 1) * jin\n",
    "    if jin == 0:\n",
    "        jin = stride\n",
    "    else:\n",
    "        jin *= stride\n",
    "offset = math.ceil(rin / jin)\n",
    "\n",
    "offset = int(offset)\n",
    "print(offset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([220]), torch.Size([220]))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "copies, bsz, dim, tsz, steps = s.shape\n",
    "steps = min(steps, tsz - offset)\n",
    "predictions = s.new(bsz * copies * (tsz - offset + 1) * steps - ((steps + 1) * steps // 2) * copies * bsz)\n",
    "labels = torch.zeros_like(predictions)\n",
    "predictions.shape, labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([11, 2, 10, 7, 12]),\n",
       " torch.Size([11, 2, 10, 7]),\n",
       " torch.Size([11, 2, 10, 4]),\n",
       " torch.Size([11, 2, 10, 4]))"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.shape, targets.shape, s[..., :-offset, i].shape, targets[..., offset:].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 8 88 3 torch.Size([11, 2, 10, 4]) torch.Size([11, 2, 10, 4])\n",
      "tensor([0., 0., 0., 0., 0., 0., 0., 0.])\n",
      "88 6 154 4 torch.Size([11, 2, 10, 3]) torch.Size([11, 2, 10, 3])\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "154 2 176 6 torch.Size([11, 2, 10, 1]) torch.Size([11, 2, 10, 1])\n",
      "tensor([0., 0.])\n",
      "176 -4 132 9 torch.Size([11, 2, 10, 0]) torch.Size([11, 2, 10, 0])\n",
      "tensor([])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1.,\n",
       "        1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0.])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start = end = 0\n",
    "for i in range(steps):\n",
    "    offset = i + offset\n",
    "    end = start + (tsz - offset) * bsz * copies\n",
    "    pos_num = (end - start) // copies\n",
    "    print(start, pos_num, end, offset, s[..., :-offset, i].shape, targets[..., offset:].shape)\n",
    "    predictions[start:end] = (s[..., :-offset, i].float() * targets[..., offset:].float()).sum(dim=2).flatten()\n",
    "    print(labels[start:start + pos_num])\n",
    "    labels[start:start + pos_num] = 1.\n",
    "    start = end\n",
    "    \n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3.66452038e-01,  3.81497800e-01,  1.17975384e-01,  3.86789769e-01,\n",
       "       -5.22472978e-01,  2.96876747e-02,  1.16604976e-02, -2.87122130e-01,\n",
       "       -5.71662426e-01,  3.81497800e-01,  1.39530897e-01,  4.34068859e-01,\n",
       "       -6.46445900e-04,  5.10506220e-02,  1.72045007e-01, -1.56508684e-02,\n",
       "        3.22836012e-01,  3.81497800e-01,  1.98190331e-01,  6.12615168e-01,\n",
       "       -5.22472978e-01,  7.26154149e-02,  1.16604976e-02, -2.87122130e-01,\n",
       "        1.93178654e-04,  3.81497800e-01,  3.89128476e-01,  3.86789769e-01,\n",
       "       -6.46445900e-04, -1.97719205e-02,  1.16604976e-02,  1.03846192e-01,\n",
       "       -3.02614093e-01, -7.46433139e-01,  4.23885345e-01,  6.11973643e-01,\n",
       "        4.69868928e-01, -1.97719205e-02,  1.72045007e-01,  1.03846192e-01,\n",
       "        1.93178654e-04,  3.81497800e-01,  4.23885345e-01,  6.12615168e-01,\n",
       "        4.69868928e-01, -1.97719205e-02,  1.72045007e-01, -1.56631157e-01,\n",
       "        3.22836012e-01,  1.09955943e+00,  1.98190331e-01,  3.86789769e-01,\n",
       "        2.98553944e-01,  5.10506220e-02,  1.72045007e-01, -3.18063974e-01,\n",
       "       -3.02614093e-01,  4.66346890e-02,  4.23885345e-01, -5.15238345e-01,\n",
       "        4.69868928e-01,  2.96876747e-02,  1.16604976e-02,  1.03846192e-01,\n",
       "       -3.02614093e-01,  3.81497800e-01,  1.98190331e-01,  6.11973643e-01,\n",
       "       -5.22472978e-01,  7.26154149e-02,  1.33439168e-01, -3.18063974e-01,\n",
       "        3.25847238e-01,  1.02858454e-01,  1.39530897e-01,  6.11973643e-01,\n",
       "       -1.59006226e+00,  5.10506220e-02,  1.72045007e-01,  4.09733653e-01,\n",
       "        3.02693009e-01,  4.66346890e-02,  1.06446333e-02,  6.11973643e-01,\n",
       "        4.69868928e-01, -3.68873119e-01,  2.99941838e-01, -1.95966244e-01,\n",
       "       -4.13644135e-01, -1.29237294e-01,  1.60333395e-01, -3.52846593e-01,\n",
       "       -1.56483725e-01,  3.66783708e-01, -4.13644135e-01, -2.71202117e-01,\n",
       "       -1.61071479e-01, -3.36294562e-01, -2.24780828e-01,  2.57270455e-01,\n",
       "       -4.13644135e-01, -1.69466317e-01, -4.43083167e-01,  5.34787297e-01,\n",
       "       -1.56483725e-01,  3.66783708e-01, -4.13644135e-01, -1.87030524e-01,\n",
       "        1.60333395e-01, -4.03155461e-02, -1.56483725e-01, -3.08349550e-01,\n",
       "       -6.05947077e-01,  2.67160714e-01,  7.26729155e-01, -4.03155461e-02,\n",
       "       -2.24780828e-01, -3.08349550e-01, -4.13644135e-01,  2.67160714e-01,\n",
       "       -4.43083167e-01, -4.03155461e-02, -2.24780828e-01,  2.38947958e-01,\n",
       "        6.00431621e-01, -1.69466317e-01,  1.60333395e-01, -3.36294562e-01,\n",
       "       -2.24780828e-01, -3.04946840e-01,  1.82850763e-01,  2.67160714e-01,\n",
       "       -1.42082041e-02, -3.52846593e-01, -1.56483725e-01, -3.08349550e-01,\n",
       "       -4.13644135e-01, -1.69466317e-01,  7.26729155e-01,  5.34787297e-01,\n",
       "        4.26046550e-02, -3.04946840e-01,  7.43078351e-01, -2.71202117e-01,\n",
       "        7.26729155e-01, -3.36294562e-01, -2.24780828e-01, -4.00821835e-01,\n",
       "        1.82850763e-01,  4.71313477e-01,  7.26729155e-01,  1.44188344e-01,\n",
       "       -3.04346308e-02, -3.80576074e-01,  3.49486321e-01,  1.49221234e-02,\n",
       "       -1.40055329e-01, -1.21881872e-01,  8.19714814e-02,  1.49221234e-02,\n",
       "        3.49486321e-01, -1.40668884e-01,  2.77836770e-01, -1.40668884e-01,\n",
       "        8.19714814e-02,  1.61352471e-01,  3.49486321e-01,  1.23742744e-02,\n",
       "        4.80696619e-01, -1.40668884e-01,  2.77836770e-01,  1.23742744e-02,\n",
       "        2.77836770e-01, -3.08042616e-02,  2.77836770e-01, -2.98715889e-01],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.detach().numpy()[:-4 * 11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.nn.functional.binary_cross_entropy(predictions, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/torch/nn/functional.py:1351: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
      "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([0.5906, 0.5942, 0.5295, 0.5955, 0.3723, 0.5074, 0.5029, 0.4287, 0.3609,\n",
       "        0.5942, 0.5348, 0.6068, 0.4998, 0.5128, 0.5429, 0.4961, 0.5800, 0.5942,\n",
       "        0.5494, 0.6485, 0.3723, 0.5181, 0.5029, 0.4287, 0.5000, 0.5942, 0.5961,\n",
       "        0.5955, 0.4998, 0.4951, 0.5029, 0.5259, 0.4249, 0.3216, 0.6044, 0.6484,\n",
       "        0.6154, 0.4951, 0.5429, 0.5259, 0.5000, 0.5942, 0.6044, 0.6485, 0.6154,\n",
       "        0.4951, 0.5429, 0.4609, 0.5800, 0.7502, 0.5494, 0.5955, 0.5741, 0.5128,\n",
       "        0.5429, 0.4211, 0.4249, 0.5117, 0.6044, 0.3740, 0.6154, 0.5074, 0.5029,\n",
       "        0.5259, 0.4249, 0.5942, 0.5494, 0.6484, 0.3723, 0.5181, 0.5333, 0.4211,\n",
       "        0.5807, 0.5257, 0.5348, 0.6484, 0.1694, 0.5128, 0.5429, 0.6010, 0.5751,\n",
       "        0.5117, 0.5027, 0.6484, 0.6154, 0.4088, 0.5744, 0.4512, 0.3980, 0.4677,\n",
       "        0.5400, 0.4127, 0.4610, 0.5907, 0.3980, 0.4326, 0.4598, 0.4167, 0.4440,\n",
       "        0.5640, 0.3980, 0.4577, 0.3910, 0.6306, 0.4610, 0.5907, 0.3980, 0.4534,\n",
       "        0.5400, 0.4899, 0.4610, 0.4235, 0.3530, 0.5664, 0.6741, 0.4899, 0.4440,\n",
       "        0.4235, 0.3980, 0.5664, 0.3910, 0.4899, 0.4440, 0.5595, 0.6458, 0.4577,\n",
       "        0.5400, 0.4167, 0.4440, 0.4243, 0.5456, 0.5664, 0.4964, 0.4127, 0.4610,\n",
       "        0.4235, 0.3980, 0.4577, 0.6741, 0.6306, 0.5106, 0.4243, 0.6777, 0.4326,\n",
       "        0.6741, 0.4167, 0.4440, 0.4011, 0.5456, 0.6157, 0.6741, 0.5360, 0.4924,\n",
       "        0.4060, 0.5865, 0.5037, 0.4650, 0.4696, 0.5205, 0.5037, 0.5865, 0.4649,\n",
       "        0.5690, 0.4649, 0.5205, 0.5403, 0.5865, 0.5031, 0.6179, 0.4649, 0.5690,\n",
       "        0.5031, 0.5690, 0.4923, 0.5690, 0.4259, 0.5061, 0.5000, 0.5000, 0.0000,\n",
       "        0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 1.0000,\n",
       "        0.0000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.9986, 1.0000,\n",
       "        0.5000, 0.5000, 0.0000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.5000,\n",
       "        0.5000, 0.5000, 0.5000, 0.5000, 0.5000, 0.3149, 0.5000, 1.0000, 1.0000,\n",
       "        0.5000, 0.0000, 0.5000, 0.5000], grad_fn=<SigmoidBackward>)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.nn.functional.sigmoid(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}