08-Vectors-and-similarity-measures.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "01219948-878e-4b10-82ce-b9d107beb583",
   "metadata": {},
   "source": [
    "# Vectors and Similarity Measures"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1aa2761-2143-428c-9762-78f4f17a72ea",
   "metadata": {},
   "source": [
    "### Comparing texts by creating vectors based on word counts and using cosine similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82e54092-813b-44aa-855a-189caf47c46e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73bf01e1-1541-4e86-89ec-24e6cf092769",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "#The files we want to vectorize\n",
    "text_files = glob.glob('soderberg-corpus/*.txt')\n",
    "print(text_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e437d6f-4a3d-4692-97be-53cdb2336f95",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converting our texts into a document-term matrix.\n",
    "#Initialize vectorizer\n",
    "cv = CountVectorizer(input='filename',\n",
    "                             lowercase=True,\n",
    "                             stop_words='english')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9339a3c-74e7-464a-9296-f38e9c33d0a1",
   "metadata": {},
   "source": [
    "N.B. This uses the default Scikit-Learn stopwords list. Try it first with the default, but you can use your own custom stopwords list if you want to modify your results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d830704d-a201-431e-a2ff-5b6b6259ce2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read in your txt file as list\n",
    "#with open('custom-stopwords.txt', 'r') as f:\n",
    "    #custom_stopwords = [s.rstrip('\\n') for s in f.readlines()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee018efb-8db9-48e0-a49a-a3871306d2c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set up tf-idf vectorizing\n",
    "#cv = CountVectorizer(input='filename' , stop_words=custom_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e63d3df1-e376-48e0-b756-5d5acc8a195c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This does the actual vectorization and creates a document term matrix\n",
    "dtm = cv.fit_transform(text_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7ca6cd6-ff21-4649-9889-8b3a41a0b254",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Shows you truncated vectors for each document/text in the corpus\n",
    "print(dtm.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "155a467a-db02-4e43-9eca-2352ef34b7fa",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Shows you all items in vocabulary with their column index \n",
    "cv.vocabulary_.items()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4b91f94-f7fb-46e8-b964-86abdd33bb2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Return total number of documents and the number of items in the vocabulary\n",
    "dc, vc = dtm.shape\n",
    "print('document count:',dc,'vocabulary count:',vc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e770da9-44c5-4dbe-a07a-7a173682fd79",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# What are our top words across all documents?\n",
    "vocab_sums = dtm.sum(axis=0)\n",
    "sorted_vocab = [(v, vocab_sums[0, i]) for v, i in cv.vocabulary_.items()]\n",
    "sorted_vocab = sorted(sorted_vocab, key = lambda x: x[1], reverse=True)\n",
    "\n",
    "# Display top twenty words\n",
    "for i in range(1,20):\n",
    "    print(sorted_vocab[i][0],\"->\",sorted_vocab[i][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8aa7bca4-fe5b-47b7-9065-1c6cb50e9704",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Assessing similarties between texts \n",
    "#based on shared term frequencies across documents\n",
    "\n",
    "# Creating a similarity matrix using cosine similarity\n",
    "cosine_matrix = cosine_similarity(dtm)\n",
    "cosine_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c559e30-9cc6-44ac-9643-5707db144264",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Creates a dataframe with cosine similarities between the texts\n",
    "#The closer to 1 the more similar\n",
    "cosine_sim = pd.DataFrame(cosine_matrix, \n",
    "                                columns=text_files, index=text_files)\n",
    "cosine_sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60f8b9a6-4c69-4a3c-83ea-3ff6652475da",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Visualize the cosine similarity between texts with heatmap \n",
    "#The closer to 1 the more similar (based on word counts)\n",
    "#i.e. texts that share similar words\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10,10))\n",
    "\n",
    "sns.heatmap(data=cosine_sim, annot=False, yticklabels=True,\n",
    "           xticklabels=True, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca719634-2edb-4c83-aea7-7774e13063f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Visualize the cosine similarity between texts with clustermap \n",
    "#Clusters together texts that are most similar\n",
    "#i.e. texts that share similar words\n",
    "\n",
    "sns.clustermap(data=cosine_sim, annot=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "075a5a60-6fc5-4e51-bed0-1848493cb148",
   "metadata": {},
   "source": [
    "### Comparing texts by creating vectors based on tf-idf scores and using cosine similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef743885-505e-4e99-bcc2-d7e1bcc9fd83",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the libraries we're going to use\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import pandas as pd\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import glob\n",
    "from pathlib import Path \n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1daf0e99-9ed4-426b-bacd-b480dcbe5bd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data: set up path to files and a variable with file names\n",
    "directory_path = 'soderberg-corpus'\n",
    "text_files = glob.glob(f'{directory_path}/*.txt')\n",
    "text_titles = [Path(text).stem for text in text_files]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1c96801-b602-4c7f-8ea4-fa6674c2f113",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set up tf-idf vectorizing with custom settings\n",
    "tfidf_vectorizer = TfidfVectorizer(input='filename', \n",
    "                                   stop_words='english')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfc0055f-33c5-43c9-b0f4-cd22f5e7f1f3",
   "metadata": {},
   "source": [
    "N.B. This uses the default Scikit-Learn stopwords list. Try it first with the default, but you can use your own custom stopwords list if you want to modify your results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a668f9c9-e664-4b3c-9f8b-9f594c552906",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read in your txt file as list\n",
    "#with open('custom-stopwords.txt', 'r') as f:\n",
    "    #custom_stopwords = [s.rstrip('\\n') for s in f.readlines()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74132321-fe08-4b34-a91d-d32c9993384f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set up tf-idf vectorizing\n",
    "#tfidf_vectorizer = TfidfVectorizer(input='filename' , stop_words=custom_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ceb44a0a-c514-46d9-8961-7c24a6d55443",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Actually do the vectorizing\n",
    "tfidf_vector = tfidf_vectorizer.fit_transform(text_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f13fada-2e7e-4327-b559-13d6e3888fbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a similarity matrix using cosine similarity\n",
    "cosine_matrix_tfidf = cosine_similarity(tfidf_vector)\n",
    "cosine_matrix_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b23fd85-2fe6-4485-9292-75f91a71c52d",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Assessing similarties between texts based on tf-idf scores\n",
    "#Creates a dataframe with cosine similarities between the texts\n",
    "#calculated from vectors of tfidf scores for each text\n",
    "cosine_sim_tfidf = pd.DataFrame(cosine_matrix_tfidf, \n",
    "                                columns=text_titles, index=text_titles)\n",
    "cosine_sim_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0aef8948-814c-4a12-8667-d2085375474f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Visualize the cosine similarities between texts with heatmap \n",
    "#based on significance scores\n",
    "#(i.e. texts that share distinctive words)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10,10))\n",
    "\n",
    "sns.heatmap(data=cosine_sim_tfidf, annot=False, yticklabels=True,\n",
    "           xticklabels=True, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4df98320-e61d-44e5-9384-a36a95da1871",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Visualize relations between texts with cluster map\n",
    "sns.clustermap(data=cosine_sim_tfidf, annot=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1eb5f5f8-a27a-4b1e-ad34-cc71b2069c3b",
   "metadata": {},
   "source": [
    "### Comparing words using word2vec and cosine similarity"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45f32761-79be-4f63-b8f8-06b468b41b37",
   "metadata": {},
   "source": [
    "For a given word, what other words share similar semantic space across the corpus?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e38195e-a67d-400f-bda0-f63938f74494",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "from pathlib import Path\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from nltk.tokenize import RegexpTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb343728-456b-4c4b-a97b-06a211fb4351",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Append all texts into a list of lists of all_docs\n",
    "directory_path = 'soderberg-corpus/'\n",
    "all_docs = []\n",
    "\n",
    "for filepath in Path(directory_path).glob(\"*.txt\"):\n",
    "    with open(filepath, 'r', encoding='utf-8') as file:\n",
    "        text = file.read()\n",
    "        all_docs.append(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afaf707c-17e7-416c-88d9-b24c29ab0a6a",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Tokenize the text into sentences (and the sentences into word tokens)\n",
    "tokenizer = RegexpTokenizer(r'\\w+')\n",
    "\n",
    "def make_sentences(list_txt):\n",
    "    all_txt = []\n",
    "    for txt in list_txt:\n",
    "        lower_txt = txt.lower()\n",
    "        sentences = sent_tokenize(lower_txt)\n",
    "        sentences = [tokenizer.tokenize(sent) for sent in sentences]\n",
    "        all_txt += sentences\n",
    "    return all_txt\n",
    "\n",
    "sentences = make_sentences(all_docs)\n",
    "sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "980191a0-2aeb-4b70-84c7-6ded67b7ae2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Traning the models\n",
    "# Try playing around with vector_size and min_count \n",
    "#to see how that affects the models\n",
    "soderberg_model = gensim.models.Word2Vec(\n",
    "    sentences,\n",
    "    min_count=5, # default is 5; ignores all words with total frequency lower than 5 \n",
    "    vector_size=150) # size of Neural Network layers; default is 100."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d988e994-a5f1-4783-8885-1aa7b0f8b0b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find nearest word vectors by cosine similarity\n",
    "#the closer to 1 the more similar\n",
    "soderberg_model.wv.most_similar('sun', topn=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3144fc43-22af-4c57-9e2e-3fd241a888ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find cosine similarity between two given word vectors\n",
    "#the closer to 1 the more similar\n",
    "print(soderberg_model.wv.similarity(w1='sun',w2='god'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79eb1ca5-8cc4-48c1-8bb8-d8d43ecf25dd",
   "metadata": {},
   "source": [
    "### Comparing words using spaCy large model and cosine similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "360d2164-7c01-4c4f-b857-4edfe5edd609",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import the libraries we need\n",
    "import spacy \n",
    "import glob\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "895c602c-307d-4ec0-b240-3aeada99b0c3",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Download the language model you're interested in\n",
    "!python -m spacy download en_core_web_lg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f17b0487-e016-426a-a71a-00ecb5eb312c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load language model\n",
    "nlp = spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "337b9003-6b48-4fb8-a92e-889bff611972",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set your filepath and define your text files\n",
    "filepath = 'soderberg-corpus/'\n",
    "text_files = glob.glob(f'{filepath}/*.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c260dbfa-7cef-4e66-8324-c5d2619d9b12",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set words you want to compare\n",
    "word1 = 'sun'.lower()\n",
    "word2 = 'god'.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99b04648-f542-4802-9601-a2c4dbb6a7d2",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "#Compare the vectors of the two words for each text in the corpus\n",
    "for text_file in text_files:\n",
    "    with open(text_file, 'r', encoding='utf-8') as file:\n",
    "        text = file.read()\n",
    "\n",
    "    # Create spacy document\n",
    "    document = nlp(text)\n",
    "\n",
    "    # Initialize tokens as None\n",
    "    sun_token = None\n",
    "    god_token = None\n",
    "\n",
    "    # Iterate through the document to find tokens\n",
    "    for token in document:\n",
    "        if sun_token is None and token.text.lower() == word1:\n",
    "            sun_token = token\n",
    "        if god_token is None and token.text.lower() == word2:\n",
    "            god_token = token\n",
    "\n",
    "        # If both tokens are found, no need to continue the loop\n",
    "        if sun_token is not None and god_token is not None:\n",
    "            break\n",
    "\n",
    "    # Check if both tokens were found in the document\n",
    "    if sun_token is not None and god_token is not None:\n",
    "        similarity_score = sun_token.similarity(god_token)\n",
    "        print(f\"Similarity between '{word1}' and '{word2}' in {text_file}: {similarity_score:.4f}\")\n",
    "    else:\n",
    "        print(f\"'{word1}' and/or '{word2}' not found in {text_file}.\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}