CCD_word_vectors.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Code for calculating cosine similarity between sentences and moving windows. Uses word2vec pretrained Google News embeddings, available from https://github.com/mmihaltz/word2vec-GoogleNews-vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyemd import emd\n",
    "from gensim.similarities import WmdSimilarity\n",
    "from gensim.models import KeyedVectors\n",
    "from gensim.test.utils import get_tmpfile\n",
    "from nltk import word_tokenize\n",
    "from nltk import sent_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "import numpy\n",
    "from sklearn.manifold import TSNE\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import spatial\n",
    "import numpy as np\n",
    "from pandas import DataFrame, Series, ExcelWriter\n",
    "import itertools\n",
    "import more_itertools\n",
    "import os\n",
    "import glob\n",
    "from sklearn.decomposition import PCA\n",
    "import pandas as pd\n",
    "from nltk import pos_tag\n",
    "from nltk.corpus import wordnet as wn\n",
    "import pickle\n",
    "stopWords = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'GoogleNews-vectors-negative300.bin.gz'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = KeyedVectors.load_word2vec_format(filename, binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# play around with vectors\n",
    "model.most_similar_cosmul(positive=['woman', 'king'], negative=['man'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' take all text files in a folder and calculate average sentence vector, then cosine between each subsequent \n",
    "sentence, mean, median and sd\n",
    "'''\n",
    "tags = ['FW','JJ','JJR','JJS', 'NN', 'NNS', 'NNP', 'NNPS', 'RB', 'RBR','RBS','VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']\n",
    "stopWords = stopwords.words('english')\n",
    "\n",
    "files_list = []\n",
    "mean_cosine_list = []\n",
    "sd_cosine_list = []\n",
    "median_cosine_list = []\n",
    "\n",
    "for file in os.listdir(path):       #for every file in the path\n",
    "    if file.endswith(\"*.txt\"):       #specify certain files\n",
    "        files_list.append(file)     #save the file's name in the list_files list\n",
    "        file_encoding = 'utf8'\n",
    "        sentence_vectors = []\n",
    "        cosines = []\n",
    "        with open(file, encoding=file_encoding, errors = 'ignore') as f:\n",
    "            text = f.read() #reads over each line of the file\n",
    "            sentences = sent_tokenize(text)\n",
    "            for sentence in sentences:\n",
    "                tokens = word_tokenize(sentence)\n",
    "                tagged = pos_tag(tokens)\n",
    "                content = [(word, tag) for (word, tag) in tagged if tag in tags]\n",
    "                content2 = [(word, tag) for (word, tag) in content if word.isalpha()] #restricts string to alphabetic characters only\n",
    "                content3 = [word for (word, tag) in content2 if word not in stopWords]\n",
    "                lemmas = [getLemma(word) for word in content3]\n",
    "                \n",
    "                sentence_vec = win_avg_feature_vector(lemmas, model=model, num_features=300, index2word_set=index2word_set)\n",
    "                sentence_vectors.append(sentence_vec)\n",
    "                 \n",
    "            for v, w in zip(sentence_vectors[:-1],sentence_vectors[1:]):\n",
    "                cos = 1 - spatial.distance.cosine(v, w)\n",
    "                cosines.append(cos)\n",
    "           \n",
    "            cosines2 = [x for x in cosines if ~np.isnan(x)]\n",
    "                \n",
    "            mean_cosine = np.mean(cosines2)\n",
    "            sd_cosine = np.std(cosines2)\n",
    "            median_cosine = np.median(cosines2)\n",
    "            \n",
    "            mean_cosine_list.append(mean_cosine)\n",
    "            sd_cosine_list.append(sd_cosine)\n",
    "            median_cosine_list.append(median_cosine)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#can check distribution\n",
    "plt.hist(cosines2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''for files only one sentence long, split sentence in half and calculate cosine between each half\n",
    "'''\n",
    "tags = ['FW','JJ','JJR','JJS', 'NN', 'NNS', 'NNP', 'NNPS', 'RB', 'RBR','RBS','VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']\n",
    "stopWords = stopwords.words('english')\n",
    "files_list = []\n",
    "mean_cosine_list = []\n",
    "sd_cosine_list = []\n",
    "median_cosine_list = []\n",
    "\n",
    "for file in os.listdir(path):       #for every file in the path\n",
    "    if file.endswith(\"*.txt\"):       #specify certain files\n",
    "        file_encoding = 'utf8'\n",
    "        files_list.append(file)\n",
    "        sentence_vectors = []\n",
    "        cosines = []\n",
    "        new_sent = []\n",
    "        with open(file, encoding=file_encoding, errors = 'ignore') as f:\n",
    "            text = f.read() #reads over each line of the file\n",
    "            sentences = sent_tokenize(text)\n",
    "            sentence_length = len(sentences)\n",
    "            if sentence_length <2:\n",
    "                for sentence in sentences:\n",
    "                    firstpart, secondpart = word_tokenize(sentence)[:len(word_tokenize(sentence))//2], word_tokenize(sentence)[len(word_tokenize(sentence))//2:]\n",
    "                    new_sent.append(' '.join(firstpart))\n",
    "                    new_sent.append(' '.join(secondpart))\n",
    "            \n",
    "            for sentence in new_sent:\n",
    "                tokens = word_tokenize(sentence)\n",
    "                tagged = pos_tag(tokens)\n",
    "                content = [(word, tag) for (word, tag) in tagged if tag in tags]\n",
    "                content2 = [(word, tag) for (word, tag) in content if word.isalpha()] #restricts string to alphabetic characters only\n",
    "                content3 = [word for (word, tag) in content2 if word not in stopWords]\n",
    "                lemmas = [getLemma(word) for word in content3]\n",
    "                \n",
    "                sentence_vec = win_avg_feature_vector(lemmas, model=model, num_features=300, index2word_set=index2word_set)\n",
    "                sentence_vectors.append(sentence_vec)\n",
    "                \n",
    "            for v, w in zip(sentence_vectors[:-1],sentence_vectors[1:]):\n",
    "                cos = 1 - spatial.distance.cosine(v, w)\n",
    "                cosines.append(cos)\n",
    "                \n",
    "            cosines2 = [x for x in cosines if ~np.isnan(x)]\n",
    "            \n",
    "    \n",
    "            mean_cosine = np.mean(cosines2)\n",
    "            sd_cosine = np.std(cosines2)\n",
    "            median_cosine = np.median(cosines2)\n",
    "            \n",
    "            mean_cosine_list.append(mean_cosine)\n",
    "            sd_cosine_list.append(sd_cosine)\n",
    "            median_cosine_list.append(median_cosine)\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# window to window average vectors and cosine\n",
    "\n",
    "tags = ['FW','JJ','JJR','JJS', 'NN', 'NNS', 'NNP', 'NNPS', 'RB', 'RBR','RBS','VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']\n",
    "stopWords = stopwords.words('english')\n",
    "\n",
    "files_list = []\n",
    "mean_cosine_list = []\n",
    "sd_cosine_list = []\n",
    "median_cosine_list = []\n",
    "\n",
    "for file in os.listdir(path):       #for every file in the path\n",
    "    if file.endswith(\"*.txt\"):       #specify certain files\n",
    "        files_list.append(file)\n",
    "        file_encoding = 'utf8'\n",
    "        window_vectors = []\n",
    "        cosines = []\n",
    "        with open(file, encoding=file_encoding, errors = 'ignore') as f:\n",
    "            text = f.read() #reads over each line of the file\n",
    "            tokens = get_tokens(text)\n",
    "            tagged = pos_tag(tokens)\n",
    "            content = [(word, tag) for (word, tag) in tagged if tag in tags]\n",
    "            content2 = [(word, tag) for (word, tag) in content if word.isalpha()] #restricts string to alphabetic characters only\n",
    "            content3 = [word for (word, tag) in content2 if word not in stopWords]\n",
    "            lemmas = [getLemma(word) for word in content3]\n",
    "            \n",
    "            windows = winderise(lemmas, 2, 1) # set window and step size here\n",
    "            \n",
    "            for window in windows:\n",
    "                window_vec = win_avg_feature_vector(window, model=model, num_features=300, index2word_set=index2word_set)\n",
    "                window_vectors.append(window_vec)\n",
    "\n",
    "            for v, w in zip(window_vectors[:-1],window_vectors[1:]):\n",
    "                cos = 1 - spatial.distance.cosine(v, w)\n",
    "                cosines.append(cos)\n",
    "                   \n",
    "            cosines2 = [x for x in cosines if ~np.isnan(x)]\n",
    "\n",
    "            mean_cosine = np.mean(cosines2)\n",
    "            sd_cosine = np.std(cosines2)\n",
    "            median_cosine = np.median(cosines2)\n",
    "\n",
    "            mean_cosine_list.append(mean_cosine)\n",
    "            sd_cosine_list.append(sd_cosine)\n",
    "            median_cosine_list.append(median_cosine)\n",
    "            \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# gensim built in average vector and distance metric, but doesn't handle words not in vocab\n",
    "\n",
    "tokens1 = get_tokens(sent1)\n",
    "tokens2 = get_tokens(sent2)\n",
    "distance = model.wv.n_similarity(v, w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def winderise(text, window=10, step=1):\n",
    "    '''\n",
    "    move a sliding window over a list. entering window and/ or step size overrides function values\n",
    "    '''\n",
    "    windows = more_itertools.windowed(text, window, step=step)\n",
    "    for window in windows:\n",
    "        yield window\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tokens(text):\n",
    "    '''\n",
    "    takes input of text string, preprocesses and returns list of tokens\n",
    "    '''\n",
    "    #textData = []\n",
    "    \n",
    "    text = text.lower()\n",
    "    tokens = word_tokenize(text)\n",
    "    tokens = [word for word in tokens if word not in stopWords]\n",
    "    tokens = [word for word in tokens if word.isalpha()] #restricts string to alphabetic characters only\n",
    "    #tokens = [getLemma(token) for token in tokens] not  - not sure if I should lemmatize?\n",
    "    #textData.append(tokens)\n",
    "    #textLength = len(tokens) #calculates total number of words in the file\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def window_vector_sums(text):\n",
    "    '''\n",
    "    returns a list which contains the summed vectors for each window. Calls winderise and vector_sum functions\n",
    "    '''\n",
    "    window_sums = []\n",
    "    windows = winderise(text, 10) # takes the input text and returns sliding windows of required size\n",
    "    for text in windows:  # for the text in each window\n",
    "        window_vectors = get_vectors(text)  # call the get_vectors function to get the vector for each word\n",
    "        window_sum = vector_sum(window_vectors)  # call the vector_sum function to sum all vectors in the window \n",
    "        window_sums.append(window_sum)  # add the summed vector to a list\n",
    "        \n",
    "    return window_sums  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cosine(summed_vectors):\n",
    "    '''\n",
    "    returns a list which is the cosine similarity between each subsequent window\n",
    "    '''\n",
    "    cosines = []\n",
    "    for v, w in zip(summed_vectors[:-1],summed_vectors[1:]):\n",
    "        cos = 1 - spatial.distance.cosine(v, w)\n",
    "        cosines.append(cos)\n",
    "    \n",
    "    return cosines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_vectors(text):\n",
    "    '''\n",
    "    finds the vector for each word in the text and adds it to a list\n",
    "    '''\n",
    "    text_vectors = []\n",
    "    for word in text:\n",
    "        if word in model.wv.vocab:\n",
    "            vector = model.wv[word]\n",
    "            text_vectors.append(vector)\n",
    "        \n",
    "    return text_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vector_sum(vectors):\n",
    "    '''\n",
    "    given a list of vectors e.g. for window or sentence, return the sum of all vectors\n",
    "    '''\n",
    "    n = len(vectors)\n",
    "    d = len(vectors[0])\n",
    "\n",
    "    #create an array initialized to 0 of the same length of the word vectors\n",
    "    s = []\n",
    "    for i in range(d):\n",
    "        s.append(0)\n",
    "    s = np.array(s)\n",
    "\n",
    "    #add each word vector to the zero vector\n",
    "    for vector in vectors:\n",
    "        s = s + np.array(vector)\n",
    "\n",
    "    return list(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getLemma(word):\n",
    "    lemma = wn.morphy(word)\n",
    "    if lemma is None:\n",
    "        return word\n",
    "    else:\n",
    "        return lemma"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}