Study and Clean Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scrape tester.ipynb             \u001b[1m\u001b[34mpoemfiles\u001b[m\u001b[m/\r\n",
      "Study Data.ipynb                poetry_foundation_scraper.py\r\n",
      "\u001b[1m\u001b[31mchromedriver\u001b[m\u001b[m*                   poetry_scraper_improved.py\r\n",
      "full_scraper.py                 poetry_scraper_improved_old.py\r\n",
      "\u001b[1m\u001b[34mglove\u001b[m\u001b[m/                          rhyme_dict_full.txt\r\n",
      "\u001b[1m\u001b[31mlyrics_billboard.csv\u001b[m\u001b[m*           rhyme_dict_simple.txt\r\n",
      "\u001b[1m\u001b[31mlyrics_large.csv\u001b[m\u001b[m*               rhyme_phones.txt\r\n",
      "\u001b[1m\u001b[31mlyrics_small.csv\u001b[m\u001b[m*               \u001b[1m\u001b[34mscraper\u001b[m\u001b[m/\r\n",
      "\u001b[1m\u001b[34mmlcourse\u001b[m\u001b[m/                       scraper.py\r\n",
      "poem_VAE.py                     small_poem_set.csv\r\n"
     ]
    }
   ],
   "source": [
    "ls # start by reading in lyrics from the small dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import csv\n",
    "import re\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data = pd.read_csv(\"lyrics_small.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist</th>\n",
       "      <th>song</th>\n",
       "      <th>link</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABBA</td>\n",
       "      <td>Ahe's My Kind Of Girl</td>\n",
       "      <td>/a/abba/ahes+my+kind+of+girl_20598417.html</td>\n",
       "      <td>Look at her face, it's a wonderful face  \\nAnd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ABBA</td>\n",
       "      <td>Andante, Andante</td>\n",
       "      <td>/a/abba/andante+andante_20002708.html</td>\n",
       "      <td>Take it easy with me, please  \\nTouch me gentl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ABBA</td>\n",
       "      <td>As Good As New</td>\n",
       "      <td>/a/abba/as+good+as+new_20003033.html</td>\n",
       "      <td>I'll never know why I had to go  \\nWhy I had t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ABBA</td>\n",
       "      <td>Bang</td>\n",
       "      <td>/a/abba/bang_20598415.html</td>\n",
       "      <td>Making somebody happy is a question of give an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ABBA</td>\n",
       "      <td>Bang-A-Boomerang</td>\n",
       "      <td>/a/abba/bang+a+boomerang_20002668.html</td>\n",
       "      <td>Making somebody happy is a question of give an...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  artist                   song                                        link  \\\n",
       "0   ABBA  Ahe's My Kind Of Girl  /a/abba/ahes+my+kind+of+girl_20598417.html   \n",
       "1   ABBA       Andante, Andante       /a/abba/andante+andante_20002708.html   \n",
       "2   ABBA         As Good As New        /a/abba/as+good+as+new_20003033.html   \n",
       "3   ABBA                   Bang                  /a/abba/bang_20598415.html   \n",
       "4   ABBA       Bang-A-Boomerang      /a/abba/bang+a+boomerang_20002668.html   \n",
       "\n",
       "                                                text  \n",
       "0  Look at her face, it's a wonderful face  \\nAnd...  \n",
       "1  Take it easy with me, please  \\nTouch me gentl...  \n",
       "2  I'll never know why I had to go  \\nWhy I had t...  \n",
       "3  Making somebody happy is a question of give an...  \n",
       "4  Making somebody happy is a question of give an...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lyrics = data.loc[:,'text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "alpha-_'\n",
      "alpha\n"
     ]
    }
   ],
   "source": [
    "w = 'alpha-_\\''\n",
    "w2 = w.replace('-_\\'', '')\n",
    "print(w)\n",
    "print(w2)\n",
    "#print(w2.isalpha())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean(w):\n",
    "    return w.replace('-','').replace('—','').replace(\"\\'\",'')\n",
    "\n",
    "def load_pronunciation_dict(simple=True):\n",
    "    filename = \"rhyme_dict_simple.txt\" if simple else \"rhyme_dict_full.txt\"\n",
    "    rhyme_dict = {}\n",
    "    with open(filename, \"r\") as file:\n",
    "        for line in file: # only add words, including characters ' - and —\n",
    "            w = line.split()[0]\n",
    "            if not w[0].isalpha() or not clean(w).isalpha() or w in rhyme_dict:\n",
    "                continue\n",
    "            rhyme_dict[w] = line.split()[1:]\n",
    "    return rhyme_dict\n",
    "\n",
    "def word_pronunciation(w, rhyme_dict):\n",
    "    if w.upper() not in rhyme_dict:\n",
    "        return w.upper()\n",
    "    s = '\\\"'\n",
    "    for c in rhyme_dict[w.upper()]:\n",
    "        s += c + \" \"\n",
    "    return s[:-1] + '\\\"'\n",
    "            \n",
    "def line_pronunciation(s, rhyme_dict): # returns pronunciation of line of words\n",
    "    if len(s) == 0:\n",
    "        return \"\"\n",
    "    split = re.split('(\\W+)', s) # remove punctuation !\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ temporarily\n",
    "    words = []\n",
    "    if not (split[0].isalpha() and split[1] == \"\\'\"):\n",
    "        words.append(split[0].upper())\n",
    "    for i in range(len(split) - 1):\n",
    "        if split[i].isalpha() and split[i+1] == \"\\'\":\n",
    "            if i + 2 < len(split) and split[i+2].isalpha():\n",
    "                words.append(split[i].upper() + \"\\'\" + split[i+2].upper())\n",
    "            else:\n",
    "                words.append(split[i].upper() + \"\\'\")\n",
    "        else:\n",
    "            words.append(split[i+1].upper())\n",
    "    p = \"\"\n",
    "    for w in words:\n",
    "        p += word_pronunciation(w, rhyme_dict)\n",
    "    return p\n",
    "\n",
    "def pronunciation(s, rhyme_dict): # returns pronunciation of paragraph of words\n",
    "    paragraph = s.split(\"\\n\")\n",
    "    if len(paragraph) == 0:\n",
    "        return \"\"\n",
    "    p = \"\"\n",
    "    for line in paragraph:\n",
    "        p += line_pronunciation(line, rhyme_dict) + \"\\n\"\n",
    "    return p[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['HH', 'AH1', 'NG', 'G', 'R', 'IY0']\n"
     ]
    }
   ],
   "source": [
    "#rhyme_dict = load_pronunciation_dict(simple=False)\n",
    "#print(rhyme_dict['HUNGRY'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"HH AH0 L OW1\"\n",
      "\"HH AH0 L OW1\" \"M AY1\" \"N EY1 M\" \"IH1 Z\" \"JH AA1 N\" \"DH AH0\" BLARGSKADARF\n",
      "\"AO1 T AH0 M\" \"M UW1 N L AY2 T\"\n",
      "\"AH0\" \"W ER1 M\" \"D IH1 G Z\" \"S AY1 L AH0 N T L IY0\"\n",
      "\"IH1 N T UW0\" \"DH AH0\" \"CH EH1 S N AH2 T\"\n"
     ]
    }
   ],
   "source": [
    "w = \"hello\"\n",
    "l = \"hello my name is John the blargskadarf\"\n",
    "p = \"Autumn moonlight\\na worm digs silently\\ninto the chestnut\"\n",
    "#print(word_pronunciation(w, rhyme_dict))\n",
    "#print(line_pronunciation(l, rhyme_dict))\n",
    "#print(pronunciation(p, rhyme_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"S EH0 N S EY1 SH AH0 N\"\n",
      "\"S T EY1 SH AH0 N\"\n",
      "\"OW0 V EY1 SH AH0 N\"\n",
      "\"AH0 F IH2 L IY0 EY1 SH AH0 N\"\n",
      "\"T R AE0 N Z L EY1 SH AH0 N\"\n"
     ]
    }
   ],
   "source": [
    "#print(word_pronunciation(\"sensation\", rhyme_dict))\n",
    "#print(word_pronunciation(\"station\", rhyme_dict))\n",
    "#print(word_pronunciation(\"ovation\", rhyme_dict))\n",
    "#print(word_pronunciation(\"affiliation\", rhyme_dict))\n",
    "#print(word_pronunciation(\"translation\", rhyme_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"L UH1 K\" \"AE1 T\" \"HH ER1\" \"F EY1 S\", \"IH1 T\"\"IH1 T S\"\"EH1 S\" \"AH0\" \"W AH1 N D ER0 F AH0 L\" \"F EY1 S\"  \n",
      "\"AH0 N D\" \"IH1 T\" \"M IY1 N Z\" \"S AH1 M TH IH0 NG\" \"S P EH1 SH AH0 L\" \"T UW1\" \"M IY1\"  \n",
      "\"L UH1 K\" \"AE1 T\" \"DH AH0\" \"W EY1\" \"DH AE1 T\" \"SH IY1\" \"S M AY1 L Z\" \"W EH1 N\" \"SH IY1\" \"S IY1 Z\" \"M IY1\"  \n",
      "\"HH AW1\" \"L AH1 K IY0\" \"K AE1 N\" \"W AH1 N\" \"F EH1 L OW0\" \"B IY1\"?  \n",
      "  \n",
      "\"SH IY1 Z\"\"EH1 S\" \"JH AH1 S T\" \"M AY1\" \"K AY1 N D\" \"AH1 V\" \"G ER1 L\", \"SH IY1\" \"M EY1 K S\" \"M IY1\" \"F IY1 L\" \"F AY1 N\"  \n",
      "\"HH UW1\" \"K UH1 D\" \"EH1 V ER0\" \"B IH0 L IY1 V\" \"DH AE1 T\" \"SH IY1\" \"K UH1 D\" \"B IY1\" \"M AY1 N\"?  \n",
      "\"SH IY1 Z\"\"EH1 S\" \"JH AH1 S T\" \"M AY1\" \"K AY1 N D\" \"AH1 V\" \"G ER1 L\", \"W IH0 TH AW1 T\" \"HH ER1\" \"AY1\"\"AY1 M\"\"EH1 M\" \"B L UW1\"  \n",
      "\"AH0 N D\" \"IH1 F\" \"SH IY1\" \"EH1 V ER0\" \"L IY1 V Z\" \"M IY1\" \"W AH1 T\" \"K UH1 D\" \"AY1\" \"D UW1\", \"W AH1 T\" \"K UH1 D\" \"AY1\" \"D UW1\"?  \n",
      "  \n",
      "\"AH0 N D\" \"W EH1 N\" \"W IY1\" \"G OW1\" \"F AO1 R\" \"AH0\" \"W AO1 K\" \"IH0 N\" \"DH AH0\" \"P AA1 R K\"  \n",
      "\"AH0 N D\" \"SH IY1\" \"HH OW1 L D Z\" \"M IY1\" \"AH0 N D\" \"S K W IY1 Z IH0 Z\" \"M AY1\" \"HH AE1 N D\"  \n",
      "\"W IY1 L\"LL \"G OW1\" \"AA1 N\" \"W AO1 K IH0 NG\" \"F AO1 R\" \"AW1 ER0 Z\" \"AH0 N D\" \"T AO1 K IH0 NG\"  \n",
      "\"AH0 B AW1 T\" \"AO1 L\" \"DH AH0\" \"TH IH1 NG Z\" \"DH AE1 T\" \"W IY1\" \"P L AE1 N\"  \n",
      "  \n",
      "\"SH IY1 Z\"\"EH1 S\" \"JH AH1 S T\" \"M AY1\" \"K AY1 N D\" \"AH1 V\" \"G ER1 L\", \"SH IY1\" \"M EY1 K S\" \"M IY1\" \"F IY1 L\" \"F AY1 N\"  \n",
      "\"HH UW1\" \"K UH1 D\" \"EH1 V ER0\" \"B IH0 L IY1 V\" \"DH AE1 T\" \"SH IY1\" \"K UH1 D\" \"B IY1\" \"M AY1 N\"?  \n",
      "\"SH IY1 Z\"\"EH1 S\" \"JH AH1 S T\" \"M AY1\" \"K AY1 N D\" \"AH1 V\" \"G ER1 L\", \"W IH0 TH AW1 T\" \"HH ER1\" \"AY1\"\"AY1 M\"\"EH1 M\" \"B L UW1\"  \n",
      "\"AH0 N D\" \"IH1 F\" \"SH IY1\" \"EH1 V ER0\" \"L IY1 V Z\" \"M IY1\" \"W AH1 T\" \"K UH1 D\" \"AY1\" \"D UW1\", \"W AH1 T\" \"K UH1 D\" \"AY1\" \"D UW1\"?\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#print(pronunciation(lyrics[0], rhyme_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['arnatt', 'at', 'at-bat', 'balyeat', 'bat', 'batt', 'batte', 'begat', 'bhatt', 'biernat', 'blatt', 'brat', 'bratt', 'catt', 'chat', 'combat', 'dat', 'delatte', 'deslatte', 'elat', 'fat', 'flat', 'flatt', 'gat', 'gatt', 'glatt', 'gnat', 'gujarat', 'hat', 'hatt', 'hnat', 'inmarsat', 'jagt', 'kat', 'katt', 'klatt', 'krat', 'kratt', 'labatt', 'landsat', 'lat', 'mat', 'matt', 'matte', 'mcnatt', 'nat', 'non-fat', 'nonfat', 'pat', 'patt', 'platt', 'platte', 'pratt', 'pratte', 'prevatt', 'prevatte', 'rat', 'rat-a-tat', 'ratte', 'sadat', 'sarratt', 'sat', 'scat', 'schadt', 'shatt', 'slaght', 'slat', 'spat', 'splat', 'spratt', 'stat', 'tat', 'that', 'tit-for-tat', 'tvsat', 'vat']\n"
     ]
    }
   ],
   "source": [
    "import pronouncing\n",
    "print(pronouncing.rhymes(\"cat\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ls ./poemfiles # next we count the number of poems we've collected that have undefined words\n",
    "poems = pd.read_csv(\"./poemfiles/poems.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GloVe Test.ipynb                poem_VAE.py\r\n",
      "Image VAE.ipynb                 \u001b[1m\u001b[34mpoemfiles\u001b[m\u001b[m/\r\n",
      "Poem LSTM.ipynb                 poetry_foundation_scraper.py\r\n",
      "Poem_LSTM.py                    poetry_scraper_improved.py\r\n",
      "Study and Clean Data.ipynb      poetry_scraper_improved_old.py\r\n",
      "\u001b[1m\u001b[31mchromedriver\u001b[m\u001b[m*                   \u001b[1m\u001b[34mresults\u001b[m\u001b[m/\r\n",
      "edited_poems.csv                rhyme_dict_full.txt\r\n",
      "full_scraper.py                 rhyme_dict_simple.txt\r\n",
      "\u001b[1m\u001b[34mglove\u001b[m\u001b[m/                          rhyme_phones.txt\r\n",
      "\u001b[1m\u001b[31mlyrics_billboard.csv\u001b[m\u001b[m*           \u001b[1m\u001b[34mscraper\u001b[m\u001b[m/\r\n",
      "\u001b[1m\u001b[31mlyrics_large.csv\u001b[m\u001b[m*               scraper.py\r\n",
      "\u001b[1m\u001b[31mlyrics_small.csv\u001b[m\u001b[m*               small_poem_set.csv\r\n",
      "\u001b[1m\u001b[34mmlcourse\u001b[m\u001b[m/\r\n"
     ]
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15652\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "data = pd.read_csv('./poemfiles/poems.csv')\n",
    "temp = data.iloc[:,3].apply(str)\n",
    "newdata = data[temp != 'nan']\n",
    "newpoemdata = newdata.iloc[:,3].apply(lambda s: s.replace('\\xa0', ' '))\n",
    "print(len(newpoemdata))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'_', '⎯', '$', '}', '—', '~', '>', '(', '?', '[', \"'\", '{', ',', '─', '#', '\"', '+', '–', '*', '!', '^', ')', '.', '\\\\', '@', ':', '`', '|', '/', '=', '‘', '’', '<', ']', '%', '-', ';', '&'}\n"
     ]
    }
   ],
   "source": [
    "import string\n",
    "string.punctuation\n",
    "valid = set()\n",
    "for c in string.punctuation: valid.add(c)\n",
    "for c in ['‘', '’', '⎯', '—', '─', '–']: valid.add(c)\n",
    "print(valid)\n",
    "\n",
    "def check_valid(s):\n",
    "    for c in s:\n",
    "        if c.isalnum() or c in valid or c.isspace():\n",
    "            pass\n",
    "        else:\n",
    "            return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2774\n"
     ]
    }
   ],
   "source": [
    "changes = []\n",
    "invalid = set()\n",
    "correct = []\n",
    "for poem in newpoemdata:\n",
    "    if not check_valid(poem):\n",
    "        newpoem = \"\"\n",
    "        for c in poem:\n",
    "            if c.isalnum() or c in valid or c.isspace(): newpoem += c\n",
    "            else: invalid.add(c)\n",
    "        changes.append(poem)\n",
    "    else:\n",
    "        correct.append(poem)\n",
    "\n",
    "print(len(changes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'»', '̄', '§', '̈', '≈', '❖', '́', '\\u200b', '™', '→', '̧', '®', '‚', '▶', '♥', '€', '•', '❏', '◀', '̃', '×', '”', '¤', '☽', '£', '¡', '·', '˚', '°', '«', '¢', '¿', '★', '\\xad', '�', '̀', '♂', '\\x9f', '\\ufeff', '“', '…', '´'}\n"
     ]
    }
   ],
   "source": [
    "print(invalid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping = [('‘', '\\''), ('’', '\\''), ('⎯', '-'), ('—', '-'), ('─', '-'), ('–', '-')]\n",
    "new_correct = []\n",
    "for poem in correct:  \n",
    "    for k, v in mapping:\n",
    "        poem = poem.replace(k, v)\n",
    "    new_correct.append(poem)\n",
    "    \n",
    "for poem in new_correct:\n",
    "    for c in poem:\n",
    "        if c in ['‘', '’', '⎯', '—', '─', '–']:\n",
    "            print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_poems = pd.DataFrame({'poems':new_correct})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>poems</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Philosophic\\nin its complex, ovoid emptiness,\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>We'd  like  to  talk  with  you  about  fear t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Wise Men will unlearn your name.\\nAbove yo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>WINTER\\nMore time is spent at the window.\\n\\nS...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>May I never be afraid\\nespecially of myself\\nb...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               poems\n",
       "0  Philosophic\\nin its complex, ovoid emptiness,\\...\n",
       "1  We'd  like  to  talk  with  you  about  fear t...\n",
       "2  The Wise Men will unlearn your name.\\nAbove yo...\n",
       "3  WINTER\\nMore time is spent at the window.\\n\\nS...\n",
       "4  May I never be afraid\\nespecially of myself\\nb..."
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_poems.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'\\'' in ['‘', '’', '⎯', '—', '─', '–']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'-' in ['‘', '’', '⎯', '—', '─', '–']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_poems.to_csv(\"clean_poems.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GloVe Test.ipynb                \u001b[1m\u001b[34mmlcourse\u001b[m\u001b[m/\r\n",
      "Image VAE.ipynb                 poem_VAE.py\r\n",
      "Poem LSTM.ipynb                 \u001b[1m\u001b[34mpoemfiles\u001b[m\u001b[m/\r\n",
      "Poem_LSTM.py                    poetry_foundation_scraper.py\r\n",
      "Study and Clean Data.ipynb      poetry_scraper_improved.py\r\n",
      "\u001b[1m\u001b[31mchromedriver\u001b[m\u001b[m*                   poetry_scraper_improved_old.py\r\n",
      "clean_poems.csv                 \u001b[1m\u001b[34mresults\u001b[m\u001b[m/\r\n",
      "edited_poems.csv                rhyme_dict_full.txt\r\n",
      "full_scraper.py                 rhyme_dict_simple.txt\r\n",
      "\u001b[1m\u001b[34mglove\u001b[m\u001b[m/                          rhyme_phones.txt\r\n",
      "\u001b[1m\u001b[31mlyrics_billboard.csv\u001b[m\u001b[m*           \u001b[1m\u001b[34mscraper\u001b[m\u001b[m/\r\n",
      "\u001b[1m\u001b[31mlyrics_large.csv\u001b[m\u001b[m*               scraper.py\r\n",
      "\u001b[1m\u001b[31mlyrics_small.csv\u001b[m\u001b[m*               small_poem_set.csv\r\n"
     ]
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "edited = pd.read_csv(\"edited_poems.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 15652 entries, 0 to 15651\n",
      "Data columns (total 4 columns):\n",
      "Author                  15652 non-null object\n",
      "Title                   15652 non-null object\n",
      "Poetry Foundation ID    15652 non-null int64\n",
      "Content                 15652 non-null object\n",
      "dtypes: int64(1), object(3)\n",
      "memory usage: 489.2+ KB\n"
     ]
    }
   ],
   "source": [
    "edited.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "data = pd.read_csv(\"clean_poems.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import string\n",
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "‘’⎯—─–\n",
      "''----\n"
     ]
    }
   ],
   "source": [
    "def clean_poem(poem):\n",
    "    temp = poem.replace('‘', '\\'')\n",
    "    temp = temp.replace('’', '\\'')\n",
    "    temp = temp.replace('⎯', '-')\n",
    "    temp = temp.replace('—', '-')\n",
    "    temp = temp.replace('─', '-')\n",
    "    temp = temp.replace('–', '-')\n",
    "    return temp\n",
    "\n",
    "s = '‘’⎯—─–'\n",
    "print(s)\n",
    "print(clean_poem(s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "series = data.iloc[:,1]\n",
    "clean_series = series.apply(clean_poem)\n",
    "\n",
    "for poem in clean_series:\n",
    "    for c in poem:\n",
    "        if c in ['‘', '’', '⎯', '—', '─', '–']:\n",
    "            print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_series.to_csv(\"../Sentence-VAE/data/ultimate_poems.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}