Word Embedding - Word2Vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "82c7c592",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec\n",
    "import gensim.downloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c8c5de6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['fasttext-wiki-news-subwords-300', 'conceptnet-numberbatch-17-06-300', 'word2vec-ruscorpora-300', 'word2vec-google-news-300', 'glove-wiki-gigaword-50', 'glove-wiki-gigaword-100', 'glove-wiki-gigaword-200', 'glove-wiki-gigaword-300', 'glove-twitter-25', 'glove-twitter-50', 'glove-twitter-100', 'glove-twitter-200', '__testing_word2vec-matrix-synopsis']\n"
     ]
    }
   ],
   "source": [
    "print(list(gensim.downloader.info()['models'].keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8fdf8173",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[================================================--] 97.4% 366.5/376.1MB downloaded"
     ]
    }
   ],
   "source": [
    "glove_vectors = gensim.downloader.load('glove-wiki-gigaword-300')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3e383221",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('facebook', 0.8349668979644775),\n",
       " ('myspace', 0.7156518697738647),\n",
       " ('youtube', 0.7003145217895508),\n",
       " ('tweets', 0.6613803505897522),\n",
       " ('blog', 0.6582605242729187),\n",
       " ('blogging', 0.6539379358291626),\n",
       " ('blogs', 0.6365774869918823),\n",
       " ('instagram', 0.6170563101768494),\n",
       " ('tumblr', 0.6059106588363647),\n",
       " ('tweet', 0.5980194807052612)]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "glove_vectors.most_similar('twitter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2b9a6b7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_vector = glove_vectors.get_vector('twitter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b15adfe2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-5.7816e-01,  1.7040e-01,  4.5315e-01, -2.9551e-01,  3.9779e-01,\n",
       "       -1.1818e-01, -3.5418e-02, -4.5607e-01,  5.8318e-02, -5.5856e-01,\n",
       "        2.5115e-01,  5.5846e-01,  6.5725e-01, -5.4000e-01,  1.1624e-01,\n",
       "       -1.6971e-01, -3.0415e-01,  1.9348e-01,  7.9338e-02,  4.7702e-01,\n",
       "        5.4305e-01,  4.8144e-01, -4.9827e-01, -4.4301e-01, -8.5195e-01,\n",
       "       -7.3448e-01, -1.5931e-01,  5.0877e-01,  6.5791e-01,  2.1944e-01,\n",
       "       -2.2480e-01,  1.5808e-01, -1.8122e-01,  7.7452e-01, -4.3319e-01,\n",
       "       -7.0192e-01, -2.4444e-01, -6.1157e-02,  4.4814e-01, -3.8710e-01,\n",
       "       -5.3779e-01,  1.1006e-01, -1.7303e-01,  6.7574e-01, -1.6612e-01,\n",
       "       -4.2205e-03,  6.3733e-02,  5.4731e-02,  1.7517e-02,  3.0798e-01,\n",
       "       -2.4405e-01,  2.7635e-01,  2.1865e-01, -3.3787e-01,  7.6142e-02,\n",
       "        1.7089e-01,  3.5796e-01, -3.8512e-01, -2.2535e-01, -9.1429e-01,\n",
       "        1.0118e-01, -3.5844e-01, -2.4257e-02,  1.2927e-01, -7.4470e-02,\n",
       "       -3.8815e-01,  3.9420e-01,  6.0092e-01,  2.1004e-01,  3.1226e-01,\n",
       "        1.9341e-01, -2.4738e-01, -3.5706e-01,  2.9637e-01,  2.9142e-01,\n",
       "       -2.4076e-01,  3.0775e-01,  6.0571e-01, -2.3866e-01,  1.9391e-01,\n",
       "       -7.8293e-01, -1.6570e-01, -1.6442e-01, -9.9144e-01,  8.9865e-02,\n",
       "        2.4198e-01,  1.7102e-01,  1.2394e-01,  7.9471e-01, -4.0902e-01,\n",
       "       -7.4319e-01, -3.6755e-01,  3.4802e-01,  2.6895e-03,  1.7925e-01,\n",
       "       -4.6295e-01, -3.9015e-01,  2.8597e-02,  6.5745e-01, -9.7256e-01,\n",
       "        7.1727e-01, -1.0000e-01, -3.7245e-02, -7.4974e-02,  1.0652e-01,\n",
       "       -1.2587e-02,  6.5671e-02, -2.9779e-01, -1.4363e-01, -4.1464e-01,\n",
       "       -3.4172e-01,  1.3924e-01, -1.1854e-01,  1.7151e-01,  2.1119e-01,\n",
       "        7.9061e-02, -3.0179e-02, -2.2055e-01,  2.8566e-01, -4.4107e-01,\n",
       "       -7.4324e-01, -4.8128e-01,  5.9606e-01,  1.8514e-01,  1.0399e-01,\n",
       "       -4.8029e-01, -2.7936e-01, -3.7832e-01,  1.6777e-01, -2.4905e-01,\n",
       "       -4.8938e-01,  4.0809e-01, -4.2099e-01,  1.5758e-01,  4.8563e-01,\n",
       "       -3.3167e-01,  2.5479e-01, -1.0598e-01,  3.0092e-01, -2.5452e-01,\n",
       "       -8.4861e-01, -2.8170e-01,  1.2912e-02, -3.1606e-01, -3.3861e-02,\n",
       "        5.5030e-01,  3.5246e-01,  7.0726e-01,  5.8685e-01,  5.2708e-01,\n",
       "        1.2374e-01, -2.1266e-01, -2.7265e-01,  1.6794e-01, -9.3602e-02,\n",
       "       -2.1291e-01,  5.9804e-01, -2.8073e-02,  2.0764e-01, -8.9536e-01,\n",
       "       -4.5652e-01,  3.5578e-02,  5.3792e-01, -2.3764e-01, -6.3136e-01,\n",
       "        3.8364e-01, -9.0696e-02,  4.0507e-01, -1.1090e-01, -1.3306e-01,\n",
       "        2.1100e-02, -1.1164e-01, -6.8447e-01, -3.0630e-01,  6.4932e-01,\n",
       "        3.6055e-01,  2.5354e-01, -4.7568e-01,  5.7113e-01,  2.3459e-01,\n",
       "       -3.1514e-01, -9.7072e-02,  9.0956e-01,  6.4923e-02, -4.0199e-01,\n",
       "       -5.1081e-01, -2.4461e-02,  4.0594e-01,  2.8181e-01, -1.0563e+00,\n",
       "        2.7317e-01,  3.9763e-01, -5.9792e-01,  6.2905e-02, -5.9581e-02,\n",
       "       -3.0849e-01,  2.3431e-01, -4.1090e-01,  5.1245e-01, -5.2931e-01,\n",
       "       -1.1153e-01, -9.7511e-01,  4.0856e-01,  7.7358e-01, -3.8849e-02,\n",
       "        1.1980e-01,  4.9160e-03, -1.2922e-01, -4.3132e-01,  8.4287e-01,\n",
       "        5.2229e-01, -8.4258e-01,  4.7344e-01, -2.0669e-01, -9.6395e-01,\n",
       "       -2.7355e-01,  2.0045e-01,  8.5087e-01,  2.2920e-01,  8.4183e-01,\n",
       "       -6.4679e-01,  9.0434e-01, -2.8244e-01,  1.8300e-01,  6.5249e-01,\n",
       "       -1.0711e-01, -2.1184e-01, -2.6399e-01, -7.3542e-02,  9.1661e-02,\n",
       "       -9.2426e-01,  3.1275e-01,  2.5011e-01, -6.0325e-01,  4.8169e-01,\n",
       "       -1.9257e-01,  1.2521e-01,  2.0366e-01, -2.5984e-01,  1.5603e-01,\n",
       "        5.2896e-01,  1.6264e-02, -6.5612e-01, -9.2582e-02,  5.1754e-01,\n",
       "        1.9775e-01, -2.2313e-01, -1.9967e-01,  4.8939e-01, -1.9443e-01,\n",
       "        2.6676e-01,  1.3417e-03,  5.5372e-02,  2.9600e-01, -4.6095e-01,\n",
       "        2.8088e-01,  2.1713e-01, -4.9859e-01, -2.1979e-01, -3.0185e-02,\n",
       "        4.4531e-01,  9.4711e-04, -1.8387e-01, -8.0642e-01, -4.1788e-01,\n",
       "        1.0614e-01, -3.2019e-01, -3.3067e-01, -9.4619e-04, -9.2143e-02,\n",
       "        1.8344e-01,  1.2281e-01, -3.6119e-01, -2.4036e-01,  1.8469e-01,\n",
       "       -4.5896e-01, -9.6463e-01, -7.9612e-01,  4.1875e-01, -3.1760e-01,\n",
       "       -1.5777e-01, -5.7504e-02,  4.8996e-02,  4.2574e-01, -3.5173e-01,\n",
       "       -8.1877e-01,  1.6474e-01, -2.0229e-01, -8.5816e-01,  9.6096e-01,\n",
       "        3.0464e-01, -1.8861e-01, -2.3286e-01, -7.5365e-02, -2.5744e-01,\n",
       "       -4.4933e-01, -1.8576e-01,  2.0167e-01, -6.2516e-01, -2.0325e-01],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a7070c85",
   "metadata": {},
   "outputs": [],
   "source": [
    "apple_vector = glove_vectors.get_vector('apple')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "29f57f92",
   "metadata": {},
   "outputs": [],
   "source": [
    "orange_vector = glove_vectors.get_vector('orange')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "179ca425",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import spatial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "47f48b7e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.32060176134109497"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - spatial.distance.cosine(orange_vector, apple_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2c326f83",
   "metadata": {},
   "outputs": [],
   "source": [
    "animal_vector = glove_vectors.get_vector('animal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0293443b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.11694557219743729"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - spatial.distance.cosine(orange_vector, animal_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1044c2c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "juice_vector = glove_vectors.get_vector('juice')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f44fcf68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.47728532552719116"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - spatial.distance.cosine(orange_vector, juice_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e2a94dd8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.441051185131073"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - spatial.distance.cosine(apple_vector, juice_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b2feb840",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.09790198504924774"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - spatial.distance.cosine(animal_vector, juice_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e8b37f9a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0735817700624466"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - spatial.distance.cosine(twitter_vector, juice_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "02d4f2a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_vector = glove_vectors.get_vector('tweet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f5a7f700",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5980194211006165"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - spatial.distance.cosine(twitter_vector, tweet_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "51db46db",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = [\n",
    "    \"The MacBook Pro is a line of Macintosh notebook computers by Apple Inc.\",\n",
    "    \"Pentium is a brand used for a series of x86 architecture-compatible microprocessors produced by Intel.\",\n",
    "    \"American football, referred to simply as football in the United States and Canada.\",\n",
    "    \"The Serie A is a professional league competition for football clubs located at the top of the Italian football league system.\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0e15bcdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "import spacy\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "\n",
    "english_stopwords = stopwords.words('english')\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "punctuation = set(string.punctuation)\n",
    "\n",
    "def data_cleaner(sentence):\n",
    "    sentence = sentence.lower()\n",
    "    for c in string.punctuation:\n",
    "        sentence = sentence.replace(c, \" \")\n",
    "    document = nlp(sentence)\n",
    "    sentence = ' '.join(token.lemma_ for token in document)\n",
    "    sentence = ' '.join(word for word in sentence.split() if word not in english_stopwords)\n",
    "    sentence = re.sub('\\d', '', sentence)\n",
    "    \n",
    "    return sentence.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "88c4c6f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def avg_vector(sentence):\n",
    "    to_remove = 0\n",
    "    vector = np.zeros(300)\n",
    "    for word in sentence:\n",
    "        if word in glove_vectors.key_to_index.keys():\n",
    "            vector += glove_vectors.get_vector(word)\n",
    "        else:\n",
    "            to_remove += 1\n",
    "    if len(sentence)== to_remove:\n",
    "        return np.zeros(300)\n",
    "        \n",
    "    return vector/(len(sentence)-to_remove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c33b5ccf",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors = []\n",
    "\n",
    "for sentence in sentences:\n",
    "    vectors.append(avg_vector(data_cleaner(sentence)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "8776c594",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([ 0.08827938,  0.2051965 ,  0.08929362, -0.371945  ,  0.104481  ,\n",
       "        -0.2298181 , -0.22203688,  0.01610125,  0.42278187, -0.75414499,\n",
       "        -0.19915662,  0.09173176,  0.4875765 , -0.47093149,  0.33541813,\n",
       "         0.2786028 , -0.169387  ,  0.019512  , -0.068304  , -0.12165512,\n",
       "         0.136586  ,  0.15210925, -0.22464688, -0.12444638,  0.0528445 ,\n",
       "        -0.12840512,  0.03025537,  0.00899975,  0.38228625, -0.18794675,\n",
       "         0.12620875, -0.092196  ,  0.13638962,  0.23949725, -0.51634549,\n",
       "        -0.26008625, -0.22530326, -0.18710937, -0.09517774, -0.19937662,\n",
       "         0.15906375,  0.35603249, -0.25223375,  0.2190975 ,  0.02374388,\n",
       "         0.09801825,  0.235172  ,  0.09902125,  0.196265  , -0.16661777,\n",
       "         0.36977375, -0.05864538,  0.23165338,  0.07521562,  0.00184699,\n",
       "        -0.08794488,  0.07157625,  0.01529837, -0.01192125, -0.040429  ,\n",
       "         0.05003775,  0.06012   , -0.1173645 ,  0.24027401, -0.08571662,\n",
       "         0.30950625, -0.48429875,  0.11095763, -0.03718   , -0.10430987,\n",
       "         0.21378875, -0.06132913,  0.26364137,  0.00645063, -0.02455013,\n",
       "         0.07803175,  0.02394491, -0.15419425,  0.02260237, -0.09948032,\n",
       "        -0.06741162, -0.14126012,  0.00660575, -0.46193949,  0.316724  ,\n",
       "         0.08448   , -0.33935875,  0.2178524 , -0.03134925, -0.09861875,\n",
       "        -0.43955287, -0.10991163, -0.30314374, -0.1842763 ,  0.19880162,\n",
       "         0.14631037, -0.32175025, -0.34895463, -0.05289587, -0.41600888,\n",
       "        -0.28175402,  0.22076763, -0.06393475, -0.30579525,  0.08462528,\n",
       "        -0.25802875,  0.02135125,  0.03304288, -0.32130862,  0.06563571,\n",
       "        -0.17969976, -0.10091   ,  0.23736675,  0.1143855 , -0.0323255 ,\n",
       "        -0.10231275, -0.05005971,  0.14683062, -0.14884912, -0.05983663,\n",
       "         0.02027   , -0.49507388,  0.2876655 , -0.31193249, -0.180008  ,\n",
       "        -0.14780038, -0.12597362,  0.44514039, -0.0531456 ,  0.24812838,\n",
       "        -0.455438  ,  0.11450388, -0.21111375,  0.24455387,  0.41644925,\n",
       "        -0.11067338,  0.433895  , -0.04514587,  0.19014037,  0.00746575,\n",
       "         0.16832375,  0.45795326,  0.01163413, -0.05536812,  0.07113277,\n",
       "         0.57126023, -0.19618375,  0.02530862,  0.09493863, -0.02465999,\n",
       "         0.33985488, -0.09109612,  0.103152  , -0.04192562,  0.12473037,\n",
       "         0.07805512, -0.1214325 ,  0.26538013, -0.04743425,  0.046547  ,\n",
       "        -0.496655  ,  0.3413632 , -0.34778625,  0.00313368, -0.09479375,\n",
       "         0.02592563, -0.08374608,  0.36138625, -0.291315  ,  0.15437963,\n",
       "        -0.39370874, -0.4192375 , -0.72778012,  0.28482987, -0.19445088,\n",
       "        -0.20508087,  0.46322213, -0.09958437,  0.02044612, -0.03451988,\n",
       "        -0.01503336,  0.04308077,  0.42365937, -0.25036249,  0.50443375,\n",
       "        -0.33883825,  0.44873874, -0.28086775,  0.25342963, -0.2272055 ,\n",
       "        -0.00861912,  0.16676712, -0.05124   ,  0.23114863, -0.137756  ,\n",
       "        -0.011985  , -0.337914  ,  0.04058762, -0.31071874, -0.00125887,\n",
       "         0.54639875, -0.23039118,  0.08156775, -0.0238725 , -0.07680987,\n",
       "        -0.01561012, -0.07940625,  0.30275375, -0.12081064, -0.100523  ,\n",
       "         0.23821449, -0.028788  , -0.01068606,  0.07264125, -0.20433312,\n",
       "         0.01942175, -0.24229112,  0.22656013, -0.148085  ,  0.10349863,\n",
       "         0.10601975,  0.3395225 , -0.453778  ,  0.16780076,  0.10593278,\n",
       "         0.41430376,  0.07498525,  0.24147813,  0.00883124, -0.15430101,\n",
       "        -0.29948838, -0.19537726, -0.08123775, -0.2405125 ,  0.4449815 ,\n",
       "        -0.143533  ,  0.2569605 ,  0.07802625, -0.257267  ,  0.13372   ,\n",
       "        -0.1233905 ,  0.160178  , -0.01708751, -0.098795  , -0.41565466,\n",
       "        -0.00999725,  0.02502738, -0.34319679,  0.05414037,  0.1136825 ,\n",
       "        -0.06540013,  0.37881925, -0.33580099,  0.15168288, -0.07106263,\n",
       "        -0.18714163,  0.28956225,  0.09313525,  0.47744   , -0.08200025,\n",
       "        -0.22410125, -0.2516225 , -0.45108337, -0.0835345 ,  0.18661275,\n",
       "         0.00871775, -0.02297483, -0.01690469,  0.18530675,  0.15410316,\n",
       "        -0.18097562,  0.267444  ,  0.15343376, -0.51432876, -0.18157394,\n",
       "         0.10299721, -0.730495  ,  0.29375625, -0.16276538, -0.09889075,\n",
       "         0.08821313,  0.03638125, -0.1891685 ,  0.33955625, -0.01770625,\n",
       "         0.00173975, -0.080225  ,  0.0609055 , -0.13065863, -0.15115762,\n",
       "         0.21526499, -0.35919687,  0.13426675,  0.176095  , -0.18380375,\n",
       "         0.27593651,  0.12960075, -0.10098284,  0.08516863, -0.102819  ]),\n",
       " array([-0.0605137 ,  0.2470543 ,  0.0135731 , -0.62259101, -0.1241536 ,\n",
       "         0.16913512, -0.1705361 ,  0.2570836 ,  0.37128901, -1.26438999,\n",
       "         0.2060172 ,  0.0201948 ,  0.0709123 , -0.1490799 ,  0.1125315 ,\n",
       "         0.00294039, -0.1088129 ,  0.05532115, -0.1067084 , -0.29004489,\n",
       "         0.01724417, -0.03111143, -0.0528396 , -0.044495  ,  0.299245  ,\n",
       "         0.282149  , -0.0890394 ,  0.0370752 , -0.07315982, -0.055368  ,\n",
       "        -0.0202853 ,  0.529364  ,  0.047509  ,  0.1080928 , -0.35891191,\n",
       "         0.07315588, -0.0863946 , -0.1224037 ,  0.0825332 ,  0.06145899,\n",
       "        -0.04881931,  0.34541931, -0.3034405 ,  0.0965393 ,  0.05843   ,\n",
       "         0.0958543 ,  0.086718  , -0.08742791,  0.1094595 , -0.235874  ,\n",
       "         0.0576327 ,  0.01240164,  0.028636  ,  0.0943633 , -0.0968825 ,\n",
       "        -0.1346679 , -0.0553212 ,  0.1224554 ,  0.0844521 ,  0.0945422 ,\n",
       "         0.0095123 , -0.0368123 ,  0.0888443 ,  0.0652496 ,  0.0694837 ,\n",
       "         0.317931  , -0.09006232,  0.0529549 ,  0.0730687 ,  0.229756  ,\n",
       "        -0.0658474 ,  0.36920409,  0.1709739 ,  0.08209099, -0.0226871 ,\n",
       "         0.0142069 ,  0.1215249 , -0.2117883 ,  0.1709657 ,  0.271984  ,\n",
       "        -0.071574  , -0.3383453 , -0.05839052, -0.2437091 ,  0.3136704 ,\n",
       "         0.1398489 , -0.26422601,  0.10259835,  0.1965295 , -0.0836493 ,\n",
       "        -0.0508823 ,  0.153519  , -0.100053  , -0.038123  ,  0.17965057,\n",
       "        -0.0773498 , -0.1105283 , -0.18557282,  0.104254  , -0.58554401,\n",
       "        -0.4079124 ,  0.2031428 ,  0.1289778 , -0.186915  , -0.1844261 ,\n",
       "        -0.142339  ,  0.1437637 ,  0.16636767, -0.543351  , -0.03045439,\n",
       "        -0.03641496, -0.0572377 , -0.0342029 , -0.02060946, -0.1829502 ,\n",
       "        -0.2757156 , -0.2366785 ,  0.06055118, -0.2600515 ,  0.092743  ,\n",
       "        -0.0344393 , -0.075068  ,  0.0085833 , -0.1168594 ,  0.0183146 ,\n",
       "         0.05505073,  0.087615  ,  0.488847  , -0.026639  ,  0.3296801 ,\n",
       "         0.032174  ,  0.1520551 , -0.0559059 ,  0.1826414 ,  0.0647939 ,\n",
       "        -0.1279372 ,  0.37911004, -0.1462952 ,  0.34029642,  0.2813102 ,\n",
       "         0.13220325,  0.248808  ,  0.12220928,  0.0210194 , -0.0586861 ,\n",
       "        -0.0231027 , -0.08068862,  0.0281076 , -0.06001149,  0.0344309 ,\n",
       "         0.20754121,  0.0581365 ,  0.1125931 , -0.160303  , -0.0379568 ,\n",
       "         0.0833704 , -0.132779  ,  0.329049  ,  0.0540186 , -0.314625  ,\n",
       "        -0.05715531,  0.15913309, -0.39473234, -0.07749109,  0.1322163 ,\n",
       "         0.2210327 ,  0.079204  ,  0.256056  , -0.302831  ,  0.18027   ,\n",
       "        -0.013264  , -0.207606  , -0.43539611,  0.2002667 ,  0.0269227 ,\n",
       "        -0.0990309 ,  0.1538439 , -0.2444985 ,  0.1541569 , -0.0999687 ,\n",
       "        -0.16186699,  0.0099028 ,  0.1893479 , -0.1674343 ,  0.3798271 ,\n",
       "        -0.12327001, -0.0209949 ,  0.10903906,  0.2429586 , -0.209516  ,\n",
       "         0.0484328 ,  0.3448345 ,  0.2607596 ,  0.3507101 , -0.12707549,\n",
       "        -0.01296245, -0.19938619,  0.233638  ,  0.1300114 , -0.1318103 ,\n",
       "         0.36096381, -0.1712788 ,  0.216645  ,  0.0987236 , -0.00591011,\n",
       "        -0.1312491 , -0.2916233 ,  0.318175  , -0.17678742, -0.11126059,\n",
       "         0.28275424,  0.0929217 ,  0.138773  ,  0.1229768 , -0.4908925 ,\n",
       "         0.23953279,  0.03454837,  0.2172984 , -0.0529155 , -0.08794857,\n",
       "        -0.3672664 ,  0.191909  , -0.1349116 ,  0.2989483 , -0.017699  ,\n",
       "         0.3292466 ,  0.0458197 ,  0.3396792 , -0.0948729 , -0.0296859 ,\n",
       "        -0.3246177 ,  0.0598546 , -0.0903357 ,  0.0692883 ,  0.12551429,\n",
       "         0.1362301 ,  0.1547952 ,  0.1355611 , -0.19632314,  0.1303618 ,\n",
       "        -0.14253924,  0.1917348 , -0.0181975 , -0.08615818, -0.63980549,\n",
       "        -0.0788395 ,  0.03428244, -0.38380699,  0.0698147 ,  0.0775109 ,\n",
       "        -0.0938483 ,  0.1682249 , -0.3942836 ,  0.0427371 , -0.25140629,\n",
       "        -0.0951703 ,  0.0469026 ,  0.0636167 ,  0.3959352 , -0.0704743 ,\n",
       "        -0.12295941, -0.25586661, -0.4266705 , -0.1507225 ,  0.24962275,\n",
       "        -0.0079923 , -0.303366  , -0.0838967 ,  0.5460138 , -0.05179164,\n",
       "        -0.0597345 ,  0.234928  , -0.0858746 , -0.2524439 , -0.2668922 ,\n",
       "         0.149162  , -1.0137894 ,  0.14553741, -0.004609  , -0.0626744 ,\n",
       "         0.1048708 ,  0.01062739, -0.1568567 ,  0.36933892, -0.15656424,\n",
       "         0.02504358, -0.389951  , -0.1058265 ,  0.10464689, -0.2117693 ,\n",
       "         0.1027109 , -0.2839362 ,  0.0592229 , -0.0129124 , -0.1293243 ,\n",
       "         0.0562974 ,  0.1369165 , -0.0695158 ,  0.4008715 ,  0.2087482 ]),\n",
       " array([ 8.63895004e-02,  3.83082516e-01,  1.28385879e-01,  6.98797493e-02,\n",
       "         9.12692519e-02,  3.20530022e-02, -4.18276243e-01,  3.43036871e-01,\n",
       "        -1.08385748e-01, -1.06285749e+00,  2.70577496e-01,  1.37092996e-01,\n",
       "        -1.22629526e-02, -4.22133757e-02,  1.89216126e-01,  1.86899251e-01,\n",
       "        -9.74774826e-03, -2.33700002e-01, -6.43456271e-02, -3.45101250e-01,\n",
       "         7.03813760e-02,  1.78123876e-01,  2.34497500e-01,  4.94765752e-01,\n",
       "        -2.40944747e-01, -7.85112567e-03, -1.05554495e-01,  7.74809992e-02,\n",
       "        -1.09433748e-01, -4.24883780e-02,  7.04963785e-02,  1.03473377e-01,\n",
       "         4.94113741e-02,  1.48715623e-01, -9.26132993e-01,  1.05078974e-01,\n",
       "         6.47281231e-02,  2.28257440e-02, -2.85409249e-01, -8.49586246e-02,\n",
       "        -3.52633754e-01, -2.19550383e-01, -3.26348115e-02,  4.43341748e-01,\n",
       "         1.73391211e-01, -7.48462486e-02,  4.21551270e-02,  2.30923759e-01,\n",
       "         5.81398765e-02,  2.03314107e-02, -1.72535722e-01, -1.18305725e-01,\n",
       "         8.64944649e-02,  1.81815426e-01, -2.02801877e-01,  1.48041133e-01,\n",
       "        -1.44838745e-01,  2.60726868e-01,  1.23205623e-01, -9.36492417e-02,\n",
       "         8.59198738e-02,  2.33765428e-01,  1.20521255e-01, -1.91559993e-01,\n",
       "        -1.72307122e-01, -3.92828738e-01, -8.89216224e-02,  3.41024118e-01,\n",
       "         1.10308998e-01, -5.47076268e-02,  5.66218638e-02, -2.34774966e-02,\n",
       "        -2.04061500e-01,  1.29535876e-01, -4.85433753e-01,  2.29685027e-02,\n",
       "        -4.79142582e-02,  3.30829873e-01, -3.16927250e-01, -3.05149984e-02,\n",
       "         4.93689880e-02, -1.89557999e-01, -8.82497490e-02, -1.37856041e-01,\n",
       "        -1.43380249e-01, -2.43524252e-01,  1.13154998e-01,  1.73597785e-02,\n",
       "        -4.46173742e-02, -8.11038492e-03, -2.84078121e-01,  1.10963622e-01,\n",
       "         2.63894124e-01,  1.01805127e-01,  1.28415494e-01,  5.36030012e-02,\n",
       "        -9.91013726e-02,  5.79102482e-02, -1.95856253e-01, -1.02747494e-01,\n",
       "         8.13843751e-02, -6.93539977e-02,  5.47349695e-03,  8.68160044e-02,\n",
       "        -4.33347481e-02,  2.01359505e-01,  5.75949932e-02,  5.78674953e-02,\n",
       "         1.99846875e-01,  4.87443384e-01,  7.73023721e-02, -1.95579254e-01,\n",
       "         1.58259387e-02, -1.62990627e-01,  5.17472964e-02, -1.41664755e-01,\n",
       "         2.90195006e-02,  1.02419127e-01,  2.98624906e-02, -1.44165870e-01,\n",
       "        -4.58904996e-02,  9.75473737e-02,  9.06728674e-02,  8.16386221e-02,\n",
       "        -1.08439252e-01,  1.57967747e-01, -2.21943755e-01,  1.50608876e-01,\n",
       "         1.54813508e-01,  9.69800152e-02, -2.97783379e-01,  1.23192498e-01,\n",
       "         1.50788001e-01, -3.11558757e-01,  1.01682497e-01, -2.81071275e-02,\n",
       "        -8.83862469e-02,  1.86236255e-01, -4.71961236e-01, -2.17323797e-02,\n",
       "         4.21709130e-02, -1.28194995e-01,  9.55987517e-02,  1.24847501e-01,\n",
       "        -2.00749002e-01,  1.48917248e-01,  1.40134876e-01, -1.78082629e-01,\n",
       "        -1.93673743e-01,  1.95618754e-01,  4.09476250e-01, -2.34930248e-01,\n",
       "        -1.19980248e-01,  1.20271263e-02,  2.47498501e-01, -1.49847502e-01,\n",
       "        -3.72880999e-01, -5.83069995e-02,  8.00286224e-02, -5.23608744e-02,\n",
       "         2.43084999e-01,  1.05443499e-01,  8.82785006e-02, -1.25478749e-01,\n",
       "         1.50971005e-01, -2.06050090e-02, -5.72445742e-02,  1.63091252e-01,\n",
       "        -2.32313997e-02, -1.19238744e-01,  2.13495749e-01,  1.13388251e-01,\n",
       "        -7.82227514e-01,  1.71045002e-01,  4.22488753e-01,  1.57706753e-01,\n",
       "        -1.36924875e-01,  1.83803928e-01, -1.14027878e-01,  4.61905388e-01,\n",
       "         5.67202501e-02, -7.42497453e-02,  1.18150501e-01,  5.47581241e-02,\n",
       "         2.45672004e-01, -3.69973241e-01, -5.43269499e-01, -7.67900646e-02,\n",
       "         9.19937585e-02, -2.25756211e-02,  5.77546586e-03,  2.22319376e-01,\n",
       "         3.73460886e-01,  5.50122897e-02, -2.54258498e-01,  4.10665004e-02,\n",
       "         5.46112508e-01, -1.97957753e-01,  2.09862493e-02,  1.97239947e-02,\n",
       "         1.20194374e+00,  4.13924996e-02, -4.78427496e-02, -1.88991285e-01,\n",
       "        -1.50193749e-01, -6.78959070e-02,  2.28166752e-01,  2.45919001e-01,\n",
       "        -6.19112467e-02,  1.75450055e-03, -8.56462521e-02,  1.00671998e-01,\n",
       "        -1.89601252e-01,  4.54387508e-01, -1.19644965e-02, -2.79294741e-02,\n",
       "        -5.18199969e-02,  1.40901251e-01, -1.95503749e-01, -7.38998756e-02,\n",
       "         1.59156252e-01, -2.52242496e-01,  3.65796520e-02, -3.71387619e-01,\n",
       "         1.38156747e-01,  1.37283752e-01,  2.22177754e-01,  4.90948763e-02,\n",
       "         1.22603486e-01, -1.73689377e-01,  1.58893629e-01,  5.14453729e-02,\n",
       "         2.70142340e-01,  5.61870048e-02, -1.17646503e-01, -1.69553729e-02,\n",
       "        -6.86700270e-03,  1.15595622e-01,  6.96787573e-02,  7.62962438e-02,\n",
       "        -2.05510997e-01,  1.86881246e-01,  3.77322497e-01, -6.14313770e-02,\n",
       "        -4.57809982e-02, -2.93522477e-02, -1.80308498e-01, -7.60596241e-02,\n",
       "         6.45662844e-03, -2.29388628e-01,  2.75068002e-01, -6.04168782e-02,\n",
       "        -1.30841247e-01,  2.20671999e-01,  1.31242450e-01, -2.53493505e-01,\n",
       "        -1.07939755e-01,  7.93895032e-02, -9.95678790e-02,  5.70462570e-02,\n",
       "        -5.12472283e-02, -1.41437504e-01,  1.74400501e-01,  2.41548257e-01,\n",
       "         1.13881752e-01, -1.58141251e-01, -2.83486746e-01,  2.52841259e-02,\n",
       "         4.54862136e-03,  7.20687464e-02,  2.70582503e-02, -1.33973622e-01,\n",
       "         4.07302510e-02, -1.52224751e-01,  2.21414627e-01,  5.03381615e-03,\n",
       "        -1.80342500e+00,  1.43590982e-02,  7.27073748e-01,  3.03493120e-01,\n",
       "        -7.57966246e-02, -1.51640087e-01,  1.92759987e-02,  1.14975374e-01,\n",
       "         1.30276249e-01,  1.21389998e-01, -9.53863743e-02,  1.65964879e-01,\n",
       "         8.87205000e-02, -5.18883718e-02,  9.40500041e-02,  6.47801234e-02,\n",
       "         7.05598705e-02,  1.13076251e-01,  1.96625876e-01,  1.09768745e-01,\n",
       "         1.05329998e-01, -1.59533503e-01,  3.82337580e-03, -3.41282750e-01]),\n",
       " array([-1.85297490e-02,  7.62040266e-01,  9.81714210e-02,  7.26385758e-02,\n",
       "         6.90167049e-02,  1.25130336e-01, -3.67578825e-01,  1.75501584e-01,\n",
       "        -1.13349725e-01, -8.20545816e-01, -4.28568373e-02,  2.80759982e-01,\n",
       "         1.73109001e-01, -9.05999728e-03, -1.12994169e-02, -2.87875012e-02,\n",
       "        -8.56006648e-02, -2.66231636e-02,  2.87826133e-02, -4.61327756e-01,\n",
       "         1.04556085e-01, -4.77358016e-03, -2.30312917e-01,  1.22316172e-01,\n",
       "        -1.08680082e-01, -1.75009925e-01, -2.66584913e-01,  4.79823499e-01,\n",
       "        -1.26890335e-01, -9.11705826e-02, -4.05644976e-02, -1.25185896e-01,\n",
       "         4.67541659e-02,  8.47781651e-02, -1.23617840e+00,  2.29176335e-01,\n",
       "         3.48032498e-01,  1.54380794e-02,  1.55652505e-02, -9.62937519e-02,\n",
       "         1.71105334e-01, -2.04204004e-01, -8.14884988e-02,  2.39714491e-01,\n",
       "         1.06707831e-01,  3.05923000e-02,  2.27819917e-01,  3.49365919e-01,\n",
       "        -1.91675501e-01,  5.99149987e-03, -1.74779166e-01, -1.43469159e-02,\n",
       "        -1.11870420e-01, -5.48837059e-02, -3.83822486e-02,  3.93149166e-01,\n",
       "        -1.01374414e-01, -7.53224983e-02,  5.39360863e-02, -2.53055788e-02,\n",
       "         1.67869667e-01, -1.07208332e-02, -1.17210221e-01, -2.75371079e-01,\n",
       "        -3.32994991e-01, -7.53000006e-02, -1.47876249e-01, -7.43038358e-02,\n",
       "         3.39724133e-02, -4.27951169e-01,  3.36485829e-01,  2.09523665e-01,\n",
       "        -4.33735249e-01, -3.15255852e-02, -3.84566835e-01,  1.42031915e-01,\n",
       "         2.43035992e-01,  3.14872495e-02, -1.39553578e-01,  7.11274985e-02,\n",
       "        -1.22819752e-01,  1.70756681e-02, -3.65312494e-02, -4.15091682e-02,\n",
       "        -9.84084158e-02,  2.74108009e-03, -3.70783149e-03,  3.18799913e-03,\n",
       "         7.84604996e-02,  4.16971333e-01,  1.16170996e-01,  3.13773332e-01,\n",
       "         3.73521689e-02,  1.39121922e-01,  3.87487498e-01,  3.26344165e-01,\n",
       "        -8.95563327e-02, -2.98160829e-01,  1.02349992e-02, -4.71551662e-01,\n",
       "        -1.00457919e-01, -1.70465832e-02, -1.61472250e-01, -9.60315006e-02,\n",
       "        -8.99858245e-03,  1.15709166e-01,  1.11903332e-01, -1.75940327e-01,\n",
       "         1.24119749e-01,  4.20426009e-01,  2.05250497e-01, -2.97725123e-01,\n",
       "         2.24721250e-03, -2.57030832e-01,  1.77329163e-01, -2.70887661e-01,\n",
       "        -6.33088332e-02, -1.22698670e-01,  3.25035844e-02, -2.83355834e-01,\n",
       "         6.23637508e-02,  2.11181331e-01,  4.32197241e-01,  8.12024983e-02,\n",
       "         3.45659132e-02,  1.44096628e-02, -1.79429086e-01,  1.44795169e-01,\n",
       "         1.11702084e-01,  2.03930500e-01, -9.62475014e-02,  8.42474988e-02,\n",
       "         3.72855822e-02,  7.50638917e-03,  2.87859914e-01,  2.25981657e-02,\n",
       "         3.13184581e-01,  2.26153744e-01, -4.01397494e-01, -1.89529167e-01,\n",
       "         1.30805001e-01,  5.23270831e-02, -7.59324977e-02,  9.13502549e-02,\n",
       "         9.70600005e-02,  7.62124181e-02,  3.29972162e-01, -2.66080331e-01,\n",
       "        -3.67156998e-01,  1.58247001e-01,  4.15925000e-01, -1.43762739e-01,\n",
       "        -1.12491664e-01, -5.67566647e-02,  4.33303333e-01, -8.80936642e-02,\n",
       "        -2.61366996e-01,  1.12641248e-01, -1.58160828e-01,  9.84332206e-02,\n",
       "         1.38375669e-01, -4.82470064e-02, -2.12521672e-01,  3.29529553e-01,\n",
       "         2.18355349e-01, -1.49307051e-01, -2.46797502e-01,  1.35376084e-01,\n",
       "         1.29219251e-01, -1.59049910e-01,  2.30154836e-02,  8.44720663e-02,\n",
       "        -9.61789176e-01,  1.32152671e-01,  9.06952530e-02,  1.47995752e-01,\n",
       "        -1.30905826e-01,  1.47657868e-02, -1.58594417e-01,  6.24907585e-01,\n",
       "         4.17300313e-01, -7.87335786e-02,  2.23677915e-01,  2.14360833e-01,\n",
       "         2.06627502e-01, -3.42844983e-01, -5.50516910e-01, -1.43557928e-02,\n",
       "         3.28271687e-02, -9.57492500e-02,  1.38982569e-01,  2.99789664e-01,\n",
       "         1.43630086e-01, -6.04931397e-02,  2.47769500e-01, -3.50116706e-03,\n",
       "        -3.05833258e-02, -6.86434026e-02,  1.73724477e-01,  3.23965823e-01,\n",
       "         1.33869092e+00,  1.85894334e-01, -1.23970810e-02, -2.47894664e-01,\n",
       "        -4.27284665e-02, -2.16301571e-01,  8.11717541e-02, -1.42991665e-01,\n",
       "         6.61252007e-02,  3.61719158e-02,  1.22667997e-01,  4.30291668e-02,\n",
       "        -3.05043424e-01,  2.27338160e-01, -2.80216640e-02, -2.40049999e-02,\n",
       "        -2.25417484e-02, -7.37944978e-02, -9.75869143e-02, -1.62271664e-01,\n",
       "         2.48891612e-03, -3.83118829e-01, -8.01664162e-02, -2.97934999e-01,\n",
       "        -1.01317161e-01, -1.18112251e-01, -2.07095664e-01,  9.94096718e-02,\n",
       "        -1.88928920e-01,  6.03386131e-03,  2.79863669e-01,  1.69971754e-01,\n",
       "         1.77001332e-01, -3.24182416e-01,  2.58498253e-01, -1.13197750e-01,\n",
       "         9.86498495e-02,  2.62130494e-01,  2.77614174e-01,  3.71404826e-01,\n",
       "         4.53153003e-01, -2.29673806e-01,  5.05668337e-01, -6.71248287e-02,\n",
       "         1.24999228e-02,  3.70606334e-01, -7.17663299e-02,  1.16742834e-01,\n",
       "        -7.08961992e-02, -5.07598216e-02,  1.77029420e-01, -1.49528338e-02,\n",
       "        -4.42157985e-02,  1.28828167e-01,  1.27241298e-01, -1.46155800e-01,\n",
       "         1.62612483e-01, -6.20912540e-02, -1.33213414e-01,  2.92818415e-01,\n",
       "        -3.07495747e-01, -2.13289257e-01,  2.48523747e-01,  1.26555085e-01,\n",
       "         2.36998335e-01, -1.26356835e-01, -3.43515829e-01,  6.30611644e-02,\n",
       "         2.59442140e-01, -2.35120996e-01,  1.14600249e-01,  9.82668639e-04,\n",
       "        -6.37283176e-03, -3.15653322e-02,  5.05684405e-01, -3.17272454e-02,\n",
       "        -1.39585417e+00,  1.97103308e-02,  3.46465827e-01,  9.69870836e-02,\n",
       "        -2.41780256e-01,  1.07403329e-02, -2.49164167e-02,  3.06845586e-01,\n",
       "         1.21621168e-01, -9.27016717e-02, -1.27246828e-01, -1.36520999e-01,\n",
       "         3.18518752e-01, -3.62624999e-01, -2.85194167e-01, -8.49755791e-04,\n",
       "        -3.78583340e-01,  7.49451633e-02,  1.98144998e-01,  9.58091610e-02,\n",
       "         2.40198330e-01, -5.01478181e-01, -4.58968324e-02, -2.45809341e-01])]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "1cf00155",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6553946566063276"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - spatial.distance.cosine(vectors[0], vectors[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e26f3930",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6928358646640873"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - spatial.distance.cosine(vectors[2], vectors[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "1a6f5265",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.20898472500966325"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - spatial.distance.cosine(vectors[0], vectors[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "1c46e449",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2559275422687064"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - spatial.distance.cosine(vectors[1], vectors[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "b8ac6f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2382190857623825"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - spatial.distance.cosine(vectors[0], vectors[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "493f21cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.23785006511457152"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - spatial.distance.cosine(vectors[1], vectors[3])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5a0053e",
   "metadata": {},
   "source": [
    "## ESERCIZIO"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c4afbab",
   "metadata": {},
   "source": [
    "## Utilizzando il dataset visto nella lezione del topic modeling, individuare il documento del dataset, più simile ad uno dei documenti a scelta dello stesso dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "7fbd11a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f23bd78a",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv('datasets/Lezione_7-Topic_modeling/dataset_Research_Article.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e7ec7c90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>TITLE</th>\n",
       "      <th>ABSTRACT</th>\n",
       "      <th>Computer Science</th>\n",
       "      <th>Physics</th>\n",
       "      <th>Mathematics</th>\n",
       "      <th>Statistics</th>\n",
       "      <th>Quantitative Biology</th>\n",
       "      <th>Quantitative Finance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Reconstructing Subject-Specific Effect Maps</td>\n",
       "      <td>Predictive models allow subject-specific inf...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Rotation Invariance Neural Network</td>\n",
       "      <td>Rotation invariance and translation invarian...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Spherical polyharmonics and Poisson kernels fo...</td>\n",
       "      <td>We introduce and develop the notion of spher...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>A finite element approximation for the stochas...</td>\n",
       "      <td>The stochastic Landau--Lifshitz--Gilbert (LL...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Comparative study of Discrete Wavelet Transfor...</td>\n",
       "      <td>Fourier-transform infra-red (FTIR) spectra o...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20967</th>\n",
       "      <td>20968</td>\n",
       "      <td>Contemporary machine learning: a guide for pra...</td>\n",
       "      <td>Machine learning is finding increasingly bro...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20968</th>\n",
       "      <td>20969</td>\n",
       "      <td>Uniform diamond coatings on WC-Co hard alloy c...</td>\n",
       "      <td>Polycrystalline diamond coatings have been g...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20969</th>\n",
       "      <td>20970</td>\n",
       "      <td>Analysing Soccer Games with Clustering and Con...</td>\n",
       "      <td>We present a new approach for identifying si...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20970</th>\n",
       "      <td>20971</td>\n",
       "      <td>On the Efficient Simulation of the Left-Tail o...</td>\n",
       "      <td>The sum of Log-normal variates is encountere...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20971</th>\n",
       "      <td>20972</td>\n",
       "      <td>Why optional stopping is a problem for Bayesians</td>\n",
       "      <td>Recently, optional stopping has been a subje...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20972 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          ID                                              TITLE  \\\n",
       "0          1        Reconstructing Subject-Specific Effect Maps   \n",
       "1          2                 Rotation Invariance Neural Network   \n",
       "2          3  Spherical polyharmonics and Poisson kernels fo...   \n",
       "3          4  A finite element approximation for the stochas...   \n",
       "4          5  Comparative study of Discrete Wavelet Transfor...   \n",
       "...      ...                                                ...   \n",
       "20967  20968  Contemporary machine learning: a guide for pra...   \n",
       "20968  20969  Uniform diamond coatings on WC-Co hard alloy c...   \n",
       "20969  20970  Analysing Soccer Games with Clustering and Con...   \n",
       "20970  20971  On the Efficient Simulation of the Left-Tail o...   \n",
       "20971  20972   Why optional stopping is a problem for Bayesians   \n",
       "\n",
       "                                                ABSTRACT  Computer Science  \\\n",
       "0        Predictive models allow subject-specific inf...                 1   \n",
       "1        Rotation invariance and translation invarian...                 1   \n",
       "2        We introduce and develop the notion of spher...                 0   \n",
       "3        The stochastic Landau--Lifshitz--Gilbert (LL...                 0   \n",
       "4        Fourier-transform infra-red (FTIR) spectra o...                 1   \n",
       "...                                                  ...               ...   \n",
       "20967    Machine learning is finding increasingly bro...                 1   \n",
       "20968    Polycrystalline diamond coatings have been g...                 0   \n",
       "20969    We present a new approach for identifying si...                 1   \n",
       "20970    The sum of Log-normal variates is encountere...                 0   \n",
       "20971    Recently, optional stopping has been a subje...                 0   \n",
       "\n",
       "       Physics  Mathematics  Statistics  Quantitative Biology  \\\n",
       "0            0            0           0                     0   \n",
       "1            0            0           0                     0   \n",
       "2            0            1           0                     0   \n",
       "3            0            1           0                     0   \n",
       "4            0            0           1                     0   \n",
       "...        ...          ...         ...                   ...   \n",
       "20967        1            0           0                     0   \n",
       "20968        1            0           0                     0   \n",
       "20969        0            0           0                     0   \n",
       "20970        0            1           1                     0   \n",
       "20971        0            1           1                     0   \n",
       "\n",
       "       Quantitative Finance  \n",
       "0                         0  \n",
       "1                         0  \n",
       "2                         0  \n",
       "3                         0  \n",
       "4                         0  \n",
       "...                     ...  \n",
       "20967                     0  \n",
       "20968                     0  \n",
       "20969                     0  \n",
       "20970                     0  \n",
       "20971                     0  \n",
       "\n",
       "[20972 rows x 9 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "73199266",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors = [avg_vector(data_cleaner(doc)) for doc in dataset['TITLE']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86364c47",
   "metadata": {},
   "outputs": [],
   "source": [
    "def most_similar(vectors, index):\n",
    "    similarity = 0\n",
    "    index_similar_doc = 0\n",
    "    for i in range(0,len(vectors)):\n",
    "        if i!=index and vectors[i].all()!=np.zeros(300).all():\n",
    "            if 1 - spatial.distance.cosine(vectors[index], vectors[i]) > similarity:\n",
    "                index_similar_doc = i\n",
    "                similarity = 1 - spatial.distance.cosine(vectors[index], vectors[i])\n",
    "    \n",
    "    return similarity,index_similar_doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fea619b",
   "metadata": {},
   "outputs": [],
   "source": [
    "index = 890\n",
    "similarity,index_similar_doc = most_similar(vectors,index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "809908a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "index_similar_doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bba264b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "735d6f31",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset['TITLE'][index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8ec166c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset['TITLE'][index_similar_doc]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "c892f89c",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'index_similar_doc' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_24192/300882871.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mindex_similar_doc\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'index_similar_doc' is not defined"
     ]
    }
   ],
   "source": [
    "index_similar_doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9560d3b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3092b8ae",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c4dfbdd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40b928db",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}