Line Generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Generation.ipynb  \u001b[1m\u001b[31mdowloaddata.sh\u001b[m\u001b[m*        ptb.valid.txt\r\n",
      "Line Generation.ipynb  \u001b[1m\u001b[34mdumps\u001b[m\u001b[m/                 ptb.vocab.json\r\n",
      "README.md              \u001b[1m\u001b[34mfigs\u001b[m\u001b[m/                  requirements.txt\r\n",
      "\u001b[1m\u001b[34m__pycache__\u001b[m\u001b[m/           inference.py           \u001b[1m\u001b[34msimple-examples\u001b[m\u001b[m/\r\n",
      "\u001b[1m\u001b[34mbin\u001b[m\u001b[m/                   model.py               simple-examples.tgz\r\n",
      "\u001b[1m\u001b[34mdata\u001b[m\u001b[m/                  ptb.py                 train.py\r\n",
      "\u001b[1m\u001b[34mdata-copy\u001b[m\u001b[m/             ptb.train.txt          utils.py\r\n"
     ]
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clean_poems.csv      glove.840B.300d.txt  ptb.vocab.json\r\n",
      "final_poems.csv      ptb.train.txt        ultimate_poems.csv\r\n",
      "glove.6B.300d.txt    ptb.valid.txt\r\n"
     ]
    }
   ],
   "source": [
    "ls data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2195884\n",
      ".\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "# prepare glove vectors\n",
    "words = set()\n",
    "word_to_vec = {}\n",
    "word_to_id = {}\n",
    "id_to_word = {}\n",
    "glove_dim = 300\n",
    "glove_filename = \"data/glove.840B.300d.txt\" # \"./glove/glove.6B.300d.txt\"\n",
    "with open(glove_filename) as file:\n",
    "    count = 0\n",
    "    for line in file:\n",
    "        tokens = line.split()\n",
    "        word, vals = tokens[0], tokens[-glove_dim:]\n",
    "        if word not in words:\n",
    "            words.add(word)\n",
    "            #word_to_vec[word] = torch.Tensor([float(val) for val in vals])\n",
    "            word_to_id[word] = count\n",
    "            id_to_word[count] = word\n",
    "            count += 1\n",
    "print(len(words))\n",
    "print(id_to_word[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en')\n",
    "filename = \"data/final_poems.csv\"\n",
    "poems = pd.read_csv(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['we', \"'d\", 'like', 'to', 'talk', 'with', 'you', 'about', 'fear', 'they', 'said', 'so', 'many', 'people', 'live', 'in', 'fear', 'these', 'days', 'they', 'drove', 'up', 'all', 'four', 'of', 'them', 'in', 'a', 'small', 'car', 'nice', 'boy', 'they', 'said', 'beautiful', 'dogs', 'they', 'said', 'so', 'friendly', 'the', 'man', 'ahead', 'of', 'the', 'woman', 'the', 'other', 'two', 'waiting', 'in', 'the', 'drive', 'i', 'was', 'outside', 'digging', 'up', 'the', 'garden', 'no', 'one', 'home', 'i', 'said', 'what', 'are', 'you', 'selling', 'anyway', 'i', \"'m\", 'not', 'interested', 'i', 'said', 'well', 'you', 'have', 'a', 'nice', 'day', 'they', 'said', 'here', \"'s\", 'our', 'card', 'there', \"'s\", 'a', 'phone', 'number', 'you', 'can', 'call', 'anytime', 'any', 'other', 'houses', 'down', 'this', 'road', 'anyone', 'else', 'live', 'here', 'we', \"'d\", 'like', 'to', 'talk', 'to', 'them', 'about', 'living', 'in', 'fear']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "poem_tokens = [word_tokenize(poem) for poem in poems.iloc[:,1]]\n",
    "print(poem_tokens[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "54838\n"
     ]
    }
   ],
   "source": [
    "vocab = set()\n",
    "for tokens in poem_tokens:\n",
    "    for token in tokens:\n",
    "        vocab.add(token)\n",
    "print(len(vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "54838\n"
     ]
    }
   ],
   "source": [
    "vocab_count = {}\n",
    "for tokens in poem_tokens:\n",
    "    for token in tokens:\n",
    "        if token not in vocab_count:\n",
    "            vocab_count[token] = 1\n",
    "        else:\n",
    "            vocab_count[token] += 1\n",
    "print(len(vocab_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8080\n"
     ]
    }
   ],
   "source": [
    "valid_vocab = set()\n",
    "for k, v in vocab_count.items():\n",
    "    if v > 10:\n",
    "        valid_vocab.add(k)\n",
    "print(len(valid_vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def valid_line(tokens, min_len, max_len, vocab):\n",
    "    if len(tokens) < 2:\n",
    "        return False\n",
    "    for token in tokens:\n",
    "        if token.isalnum() and token not in vocab:\n",
    "            return False\n",
    "        for c in token:\n",
    "            if ord(c) >= 128:\n",
    "                return False\n",
    "    return True\n",
    "\n",
    "poem_lines = []\n",
    "for poem in poems.iloc[:,1]:\n",
    "    temp_lines = poem.split('\\n')\n",
    "    for line in temp_lines:\n",
    "        tokens = word_tokenize(line)\n",
    "        if valid_line(tokens, 5, 20, valid_vocab): # valid_vocab vs words\n",
    "            poem_lines.append(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "132788\n"
     ]
    }
   ],
   "source": [
    "print(len(poem_lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shorten(lines, minlen, maxlen, vocab):\n",
    "    newlines = []\n",
    "    for line in lines: \n",
    "        if len(line) < minlen or len(line) > maxlen:\n",
    "            continue\n",
    "        for w in line:\n",
    "            if w not in vocab:\n",
    "                continue\n",
    "        newlines.append(line)\n",
    "    return newlines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "132788\n",
      "12788\n"
     ]
    }
   ],
   "source": [
    "print(len(poem_lines))\n",
    "cut = 120000\n",
    "train = poem_lines[:cut]\n",
    "valid = poem_lines[cut:]\n",
    "print(len(valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "for here the calendar breeds nights \n"
     ]
    }
   ],
   "source": [
    "def list_to_line(l):\n",
    "    output = \"\"\n",
    "    for w in l:\n",
    "        output += w + \" \"\n",
    "    return output\n",
    "\n",
    "print(list_to_line(train[15]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/ptb.train.txt\", \"w\") as file:\n",
    "    for s in train:\n",
    "        file.write(list_to_line(s))\n",
    "        file.write('\\n')\n",
    "with open(\"data/ptb.valid.txt\", \"w\") as file:\n",
    "    for s in valid:\n",
    "        file.write(list_to_line(s))\n",
    "        file.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 0\n",
    "w2i = {}\n",
    "i2w = {}\n",
    "previous = set()\n",
    "special = [\"<pad>\", \"<unk>\", \"<sos>\", \"<eos>\"]\n",
    "for token in special:\n",
    "    w2i[token] = count\n",
    "    i2w[count] = token\n",
    "    count += 1\n",
    "for tokens in poem_tokens:\n",
    "    for token in tokens:\n",
    "        if token not in previous:\n",
    "            w2i[token] = count\n",
    "            i2w[count] = token\n",
    "            previous.add(token)\n",
    "            count += 1\n",
    "final_vocab = {'w2i':w2i, 'i2w':i2w}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/ptb.vocab.json', 'w') as file:\n",
    "    json.dump(final_vocab, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clean_poems.csv      glove.840B.300d.txt  ptb.vocab.json\r\n",
      "final_poems.csv      ptb.train.txt        ultimate_poems.csv\r\n",
      "glove.6B.300d.txt    ptb.valid.txt\r\n"
     ]
    }
   ],
   "source": [
    "ls data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}