preprocess_corpus/generate-MPII-MD-corpus.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate corpus and gruond-truth references of released videos\n",
    "\n",
    "### Corpus file contents\n",
    "0. train_data: captions and idxs of training videos in format [corpus_widxs, vidxs, corpus_pidxs], where:\n",
    "    - corpus_widxs is a list of lists with the index of words in the vocabulary\n",
    "    - vidxs is a list of indexes of video features in the features file\n",
    "    - corpus_pidxs is a list of lists with the index of POS tags in the POS tagging vocabulary\n",
    "1. val_data: same format of train_data.\n",
    "2. test_data: same format of train_data.\n",
    "3. vocabulary: in format {'word': count}.\n",
    "4. idx2word: is the vocabulary in format {idx: 'word'}.\n",
    "5. word_embeddings: are the vectors of each word. The i-th row is the word vector of the i-th word in the vocabulary.\n",
    "6. idx2pos: is the vocabulary of POS tagging in format {idx: 'POSTAG'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate split for training and validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>video-id</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0001_American_Beauty_00.00.51.926-00.00.54.129</td>\n",
       "      <td>Her mind wanders for a beat.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0001_American_Beauty_00.00.56.224-00.01.03.394</td>\n",
       "      <td>Someone looks at us and sits up.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0001_American_Beauty_00.01.14.635-00.01.36.380</td>\n",
       "      <td>We are FLYING above suburban America, DESCENDI...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0001_American_Beauty_00.01.37.227-00.01.38.586</td>\n",
       "      <td>We are looking down at a king-sized BED from O...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0001_American_Beauty_00.01.38.586-00.01.40.722</td>\n",
       "      <td>An irritating ALARM CLOCK RINGS.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68370</th>\n",
       "      <td>2054_Harry_Potter_and_the_prisoner_of_azkaban_...</td>\n",
       "      <td>Someone is gone.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68371</th>\n",
       "      <td>2054_Harry_Potter_and_the_prisoner_of_azkaban_...</td>\n",
       "      <td>Someone stands amid a circle of excited Gryffi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68372</th>\n",
       "      <td>2054_Harry_Potter_and_the_prisoner_of_azkaban_...</td>\n",
       "      <td>As Someone arrives, he glances at Someone, who...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68373</th>\n",
       "      <td>2054_Harry_Potter_and_the_prisoner_of_azkaban_...</td>\n",
       "      <td>The others turn, begin all speaking at once.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68374</th>\n",
       "      <td>2054_Harry_Potter_and_the_prisoner_of_azkaban_...</td>\n",
       "      <td>Mystified, Someone steps forward, the boys pee...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>68375 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                video-id  \\\n",
       "0         0001_American_Beauty_00.00.51.926-00.00.54.129   \n",
       "1         0001_American_Beauty_00.00.56.224-00.01.03.394   \n",
       "2         0001_American_Beauty_00.01.14.635-00.01.36.380   \n",
       "3         0001_American_Beauty_00.01.37.227-00.01.38.586   \n",
       "4         0001_American_Beauty_00.01.38.586-00.01.40.722   \n",
       "...                                                  ...   \n",
       "68370  2054_Harry_Potter_and_the_prisoner_of_azkaban_...   \n",
       "68371  2054_Harry_Potter_and_the_prisoner_of_azkaban_...   \n",
       "68372  2054_Harry_Potter_and_the_prisoner_of_azkaban_...   \n",
       "68373  2054_Harry_Potter_and_the_prisoner_of_azkaban_...   \n",
       "68374  2054_Harry_Potter_and_the_prisoner_of_azkaban_...   \n",
       "\n",
       "                                                sentence  \n",
       "0                           Her mind wanders for a beat.  \n",
       "1                       Someone looks at us and sits up.  \n",
       "2      We are FLYING above suburban America, DESCENDI...  \n",
       "3      We are looking down at a king-sized BED from O...  \n",
       "4                       An irritating ALARM CLOCK RINGS.  \n",
       "...                                                  ...  \n",
       "68370                                   Someone is gone.  \n",
       "68371  Someone stands amid a circle of excited Gryffi...  \n",
       "68372  As Someone arrives, he glances at Someone, who...  \n",
       "68373       The others turn, begin all speaking at once.  \n",
       "68374  Mystified, Someone steps forward, the boys pee...  \n",
       "\n",
       "[68375 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "# data = pd.read_csv('../../../data/MPII-MD/annotations-original.csv', '\\t', usecols=[0,1], names=['video-id', 'sentence'], engine='python')  \n",
    "data = pd.read_csv('../../../data/MPII-MD/annotations-someone.csv', '\\t', usecols=[0,1], names=['video-id', 'sentence'], engine='python')  \n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list(data['sentence'])\n",
    "\n",
    "train_vidxs, train_corpus = list(data['video-id']), list(data['sentence'])\n",
    "# valid_vidxs, valid_corpus = zip(*[(int(d['id']), d['label']) for d in valid_data])\n",
    "# test_vidxs = [(int(d['id'])) for d in test_data]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get pretrained embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "400000\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "\n",
    "wordvectors = {}\n",
    "# with open('./glove.42B.300d.txt') as f:\n",
    "with open('./glove.6B.300d.txt') as f:\n",
    "    for line in f:\n",
    "        s = line.strip().split(' ')\n",
    "        if len(s) == 301:\n",
    "            wordvectors[s[0]] = np.array(s[1:], dtype=float)\n",
    "    print(len(wordvectors))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine the vocabulary from train split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/jeperez/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg. count of words per caption: 11.049681901279708\n",
      "Count of unique words:  20936\n",
      "missing word: king-sized\n",
      "missing word: fogged-up\n",
      "missing word: well-put\n",
      "missing word: color-coordinated\n",
      "missing word: digicam\n",
      "missing word: zen-like\n",
      "missing word: much-younger\n",
      "missing word: cheap-looking\n",
      "missing word: well-rehearsed\n",
      "missing word: spartanettes\n",
      "missing word: cleaver-ish\n",
      "missing word: d'eouvres\n",
      "missing word: state-of-the-\n",
      "missing word: cup-sized\n",
      "missing word: zip-loc\n",
      "missing word: post-sex\n",
      "missing word: half-smoked\n",
      "missing word: self-\n",
      "missing word: glassy-eyed\n",
      "missing word: fresh-cut\n",
      "missing word: spinning-teacup\n",
      "missing word: street-freak\n",
      "missing word: de-humidifying\n",
      "missing word: throw-up\n",
      "missing word: slack-jawed\n",
      "missing word: sweet-faced\n",
      "missing word: o.s\n",
      "missing word: half-laugh\n",
      "missing word: almost-crowded\n",
      "missing word: washclothes\n",
      "missing word: tearstained\n",
      "missing word: dress/bathrobe\n",
      "missing word: half-whisper\n",
      "missing word: begowned\n",
      "missing word: salmon-colored\n",
      "missing word: kepis\n",
      "missing word: semi-darkness\n",
      "missing word: leuchtags\n",
      "missing word: wonderingly\n",
      "missing word: chain-lights\n",
      "missing word: countersigns\n",
      "missing word: break-neck\n",
      "missing word: snow-\n",
      "missing word: ceiling-high\n",
      "missing word: sun-\n",
      "missing word: deck-chairs\n",
      "missing word: near-shock\n",
      "missing word: all-too-\n",
      "missing word: 'drawer\n",
      "missing word: velvet-corduroy\n",
      "missing word: bulldurham\n",
      "missing word: forty-odd\n",
      "missing word: consumptive-looking\n",
      "missing word: tortoise-framed\n",
      "missing word: silver-framed\n",
      "missing word: waxpaper\n",
      "missing word: wine-cup\n",
      "missing word: still-lit\n",
      "missing word: twin-pronged\n",
      "missing word: 'phone\n",
      "missing word: unknit\n",
      "missing word: tie-pin\n",
      "missing word: sandwich-man\n",
      "missing word: emptily\n",
      "missing word: slate-covered\n",
      "missing word: bear-hold\n",
      "missing word: bedstand\n",
      "missing word: tip-toes\n",
      "missing word: unbolts\n",
      "missing word: nail-\n",
      "missing word: watch-band\n",
      "missing word: b.g\n",
      "missing word: scotch-taped\n",
      "missing word: 1.children\n",
      "missing word: 2.rond\n",
      "missing word: champs-elys\n",
      "missing word: 3.children\n",
      "missing word: 4.restaurant\n",
      "missing word: 5.balloon\n",
      "missing word: p.o.v\n",
      "missing word: mid-stage\n",
      "missing word: unbelievingly\n",
      "missing word: overleafs\n",
      "missing word: overleaf\n",
      "missing word: redfaced\n",
      "missing word: sergeants-at-arms\n",
      "missing word: swayback\n",
      "missing word: heart-\n",
      "missing word: gray-haired\n",
      "missing word: a.c.below\n",
      "missing word: half-jacket\n",
      "missing word: over-flowing\n",
      "missing word: runnning\n",
      "missing word: glass-topped\n",
      "missing word: pie-\n",
      "missing word: wing-tipped\n",
      "missing word: 'the\n",
      "missing word: hatefully\n",
      "missing word: '6000\n",
      "missing word: mulvihlll\n",
      "missing word: tear-stained\n",
      "missing word: clothesmen\n",
      "missing word: musses\n",
      "missing word: towel-dressed\n",
      "missing word: once-empty\n",
      "missing word: sickly-looking\n",
      "missing word: fake-hitting\n",
      "missing word: o.c\n",
      "missing word: warningly\n",
      "missing word: video-anxious\n",
      "missing word: open-mouthed\n",
      "missing word: centerfold-a\n",
      "missing word: half-spinning\n",
      "missing word: stretcher-bound\n",
      "missing word: someone-shrieking-sprays\n",
      "missing word: ripped-open\n",
      "missing word: sugar-\n",
      "missing word: h-a-t-e\n",
      "missing word: turretted\n",
      "missing word: model-t\n",
      "missing word: sideyard\n",
      "missing word: half-falls\n",
      "missing word: blood-\n",
      "missing word: b.g.the\n",
      "missing word: l-o-v-e\n",
      "missing word: perambulator\n",
      "missing word: rose-and-gold\n",
      "missing word: tree-shaded\n",
      "missing word: one-street\n",
      "missing word: second-\n",
      "missing word: school-children\n",
      "missing word: sweep-hand\n",
      "missing word: shop-\n",
      "missing word: lollypops\n",
      "missing word: toy-like\n",
      "missing word: river-bank\n",
      "missing word: gas-lamp\n",
      "missing word: tree-shadows\n",
      "missing word: wharf-boat\n",
      "missing word: fine-looking\n",
      "missing word: door-glass\n",
      "missing word: soda-fountain\n",
      "missing word: dandles\n",
      "missing word: palely\n",
      "missing word: pebble-skimming\n",
      "missing word: o.s.from\n",
      "missing word: slantwise\n",
      "missing word: lamplighted\n",
      "missing word: shooting-frame\n",
      "missing word: gawks\n",
      "missing word: spic-and-span\n",
      "missing word: bewilderedly\n",
      "missing word: hand-clasp\n",
      "missing word: grape-arbor\n",
      "missing word: paperdoll\n",
      "missing word: hundred-dollar\n",
      "missing word: paper-doll\n",
      "missing word: watch-chain\n",
      "missing word: paper-dolls\n",
      "missing word: wedding-night\n",
      "missing word: weak-defiance\n",
      "missing word: tautens\n",
      "missing word: unconcernedly\n",
      "missing word: coal-heap\n",
      "missing word: shelf-prop\n",
      "missing word: clop-clops\n",
      "missing word: coal-dust\n",
      "missing word: snivel\n",
      "missing word: guck\n",
      "missing word: half-rises\n",
      "missing word: pokeberry\n",
      "missing word: un-moving\n",
      "missing word: hands-ful\n",
      "missing word: unsnarls\n",
      "missing word: re-sets\n",
      "missing word: bird-cage\n",
      "missing word: window-light\n",
      "missing word: sand-bar\n",
      "missing word: berry-\n",
      "missing word: rifle-target\n",
      "missing word: napes\n",
      "missing word: market-basket\n",
      "missing word: neon-lighted\n",
      "missing word: gun-barrel\n",
      "missing word: window-glass\n",
      "missing word: sportily\n",
      "missing word: coffee-pot\n",
      "missing word: pistol-\n",
      "missing word: fifty-odd\n",
      "missing word: pot-holder\n",
      "missing word: costume-jewelry\n",
      "missing word: flower-\n",
      "missing word: mid-sixtyish\n",
      "missing word: glassed-in\n",
      "missing word: truck-stop\n",
      "missing word: morning-show\n",
      "missing word: unguent\n",
      "missing word: scrape-scrape-scrape\n",
      "missing word: wind-shield\n",
      "missing word: flannel-shirt\n",
      "missing word: bill-fold\n",
      "missing word: leg-well\n",
      "missing word: pajama-clad\n",
      "missing word: big-bellied\n",
      "missing word: impedimenta\n",
      "missing word: toplit\n",
      "missing word: s-curves\n",
      "missing word: uncomprehendingly\n",
      "missing word: stucked\n",
      "missing word: lowlit\n",
      "missing word: paunching\n",
      "missing word: bank-wrapped\n",
      "missing word: carbon-copy\n",
      "missing word: armpatch\n",
      "missing word: gum-balls\n",
      "missing word: upholsters\n",
      "missing word: silhouete\n",
      "missing word: oaklined\n",
      "missing word: all-\n",
      "missing word: picure\n",
      "missing word: gear/\n",
      "missing word: chest-deep\n",
      "missing word: someone-\n",
      "missing word: cobble-stone\n",
      "missing word: relocks\n",
      "missing word: all-risk\n",
      "missing word: dry-mopping\n",
      "missing word: l.schwartz\n",
      "missing word: unbuttons\n",
      "missing word: craperoo\n",
      "missing word: bath-towel\n",
      "missing word: sweat-soaked\n",
      "missing word: normandie-\n",
      "missing word: window-pane\n",
      "missing word: glass-and-\n",
      "missing word: ice-box\n",
      "missing word: hot-\n",
      "missing word: checker-board\n",
      "missing word: low-spoken\n",
      "missing word: redcaps\n",
      "missing word: jr.president\n",
      "missing word: dictograph\n",
      "missing word: dead-pan\n",
      "missing word: untidily\n",
      "missing word: waste-paper\n",
      "missing word: close-cropped\n",
      "missing word: nut-brown\n",
      "missing word: half-\n",
      "missing word: weapon-carrier\n",
      "missing word: anglo-\n",
      "missing word: keralans\n",
      "missing word: bemedalled\n",
      "missing word: colorlessly\n",
      "missing word: harrumphs\n",
      "missing word: extemporizing\n",
      "missing word: pranam\n",
      "missing word: indolently\n",
      "missing word: disconcertedly\n",
      "missing word: toilett\n",
      "missing word: appraisingly\n",
      "missing word: hostilely\n",
      "missing word: mother-wife\n",
      "missing word: imperturbably\n",
      "missing word: canvas-topped\n",
      "missing word: challengingly\n",
      "missing word: half-listening\n",
      "missing word: well-\n",
      "missing word: flower-covered\n",
      "missing word: hollow-faced\n",
      "missing word: penetratingly\n",
      "missing word: self-deprecatingly\n",
      "missing word: aloofly\n",
      "missing word: re-directs\n",
      "missing word: ashramite\n",
      "missing word: half-starved\n",
      "missing word: ill-lit\n",
      "missing word: illumines\n",
      "missing word: white-garbed\n",
      "missing word: irritatedly\n",
      "missing word: at-ease\n",
      "missing word: desolately\n",
      "missing word: bullet-ripped\n",
      "missing word: flitters\n",
      "missing word: suitcoats\n",
      "missing word: loin-\n",
      "missing word: tail-enders\n",
      "missing word: refolds\n",
      "missing word: half-fortresses\n",
      "missing word: arms-akimbo\n",
      "missing word: ashramites\n",
      "missing word: lathis\n",
      "missing word: stockade-like\n",
      "missing word: dharasana\n",
      "missing word: under-cranked\n",
      "missing word: possessively\n",
      "missing word: black-ribboned\n",
      "missing word: half-gray\n",
      "missing word: eyeline\n",
      "missing word: goondas\n",
      "missing word: getting-up\n",
      "missing word: 'someone\n",
      "missing word: aronud\n",
      "missing word: briefrase\n",
      "missing word: tanya/marlene\n",
      "missing word: 'now\n",
      "missing word: dead-eyed\n",
      "missing word: 'right\n",
      "missing word: i.d\n",
      "missing word: arross\n",
      "missing word: dark-shaded\n",
      "missing word: onesteps\n",
      "missing word: gleamy\n",
      "missing word: 'look\n",
      "missing word: pain-killer\n",
      "missing word: hubub\n",
      "missing word: suitcoat\n",
      "missing word: jack-o-lantern\n",
      "missing word: eye-holes\n",
      "missing word: night-table\n",
      "missing word: bright-eyed\n",
      "missing word: tough-looking\n",
      "missing word: wild-\n",
      "missing word: tossled\n",
      "missing word: closed-down\n",
      "missing word: station/diner\n",
      "missing word: tromps\n",
      "missing word: totebag\n",
      "missing word: f.g\n",
      "missing word: two-\n",
      "missing word: ferocious-looking\n",
      "missing word: horror-\n",
      "missing word: horror-thon\n",
      "missing word: whishing\n",
      "missing word: tomb-like\n",
      "missing word: instictively\n",
      "missing word: unhooks\n",
      "missing word: mid-section\n",
      "missing word: bleary-eyed\n",
      "missing word: waspish-looking\n",
      "missing word: shirt-sleeves\n",
      "missing word: throne-like\n",
      "missing word: hold-frame\n",
      "missing word: weather-beaten\n",
      "missing word: guileful\n",
      "missing word: interestedly\n",
      "missing word: simply-dressed\n",
      "missing word: still-silent\n",
      "missing word: distinguished-looking\n",
      "missing word: sophisticated-looking\n",
      "missing word: u.s.o\n",
      "missing word: u.s.n\n",
      "missing word: agitatedly\n",
      "missing word: gas-lit\n",
      "missing word: coat-tails\n",
      "missing word: tollkeeper\n",
      "missing word: interrogatively\n",
      "missing word: human-propelled\n",
      "missing word: sightlessly\n",
      "missing word: reek-a\n",
      "missing word: grimy-faced\n",
      "missing word: sour-looking\n",
      "missing word: artlessly\n",
      "missing word: tin-starred\n",
      "missing word: hayport\n",
      "missing word: rat-a-tat-a-tat-\n",
      "missing word: chinka-chinka-\n",
      "missing word: chinka\n",
      "missing word: hogwallop\n",
      "missing word: white-robed\n",
      "missing word: belly-deep\n",
      "missing word: back-down\n",
      "missing word: incantates\n",
      "missing word: forebearing\n",
      "missing word: go-to-meetin\n",
      "missing word: cement-block\n",
      "missing word: hundred-foot\n",
      "missing word: 'man\n",
      "missing word: sixty-year-old\n",
      "missing word: wheedles\n",
      "missing word: babylike\n",
      "missing word: stakebed\n",
      "missing word: scratchily\n",
      "missing word: sad-faced\n",
      "missing word: langor\n",
      "missing word: xes\n",
      "missing word: powder-blue\n",
      "missing word: moss-dripping\n",
      "missing word: blearily\n",
      "missing word: pitter-\n",
      "missing word: hollow-\n",
      "missing word: bunting-covered\n",
      "missing word: pencil-\n",
      "missing word: pencil-neck\n",
      "missing word: bellyflops\n",
      "missing word: tinnily\n",
      "missing word: restively\n",
      "missing word: overarticulating\n",
      "missing word: quizically\n",
      "missing word: long-armed\n",
      "missing word: pincering\n",
      "missing word: crabwalk\n",
      "missing word: backmost\n",
      "missing word: up-\n",
      "missing word: creeeeeek\n",
      "missing word: whoooosh\n",
      "missing word: balled-up\n",
      "missing word: musical-instrument\n",
      "missing word: wharvey\n",
      "missing word: buck-and-wing\n",
      "missing word: riot-ready\n",
      "missing word: slobberier\n",
      "missing word: shotgun-wielding\n",
      "missing word: fresh-\n",
      "missing word: sepia-tinted\n",
      "missing word: up-bob\n",
      "missing word: roof-\n",
      "missing word: cottonhouse\n",
      "missing word: half-submerged\n",
      "missing word: 'angel\n",
      "missing word: podgy\n",
      "missing word: froing\n",
      "missing word: symche\n",
      "missing word: dolek\n",
      "missing word: worktop\n",
      "missing word: washers-up\n",
      "missing word: tail-gate\n",
      "missing word: cottar\n",
      "missing word: 'children\n",
      "missing word: gebcznyski\n",
      "missing word: bed-sitting\n",
      "missing word: greaseproof\n",
      "missing word: sentry-box\n",
      "missing word: iheydle\n",
      "missing word: tin-opener\n",
      "missing word: hacienda-style\n",
      "missing word: caught-off-guard\n",
      "missing word: preppy-looking\n",
      "missing word: blow-dry\n",
      "missing word: ass-kicking\n",
      "missing word: sqaure\n",
      "missing word: hellsapopinish\n",
      "missing word: tango-\n",
      "missing word: k.d\n",
      "missing word: air-guitar\n",
      "missing word: fancy-style\n",
      "missing word: someone-made\n",
      "missing word: unbeknownst-to-her\n",
      "missing word: dust-buster\n",
      "missing word: tee-shirt\n",
      "missing word: see-no-\n",
      "missing word: hear-no-evil\n",
      "missing word: speak-no-evil\n",
      "missing word: 35ish\n",
      "missing word: zzzzzz\n",
      "missing word: auditorium-\n",
      "missing word: babytalk\n",
      "missing word: broad-sided\n",
      "missing word: z-28\n",
      "missing word: pumpaction\n",
      "missing word: m-style\n",
      "missing word: hillbillys\n",
      "missing word: tuxedo-clad\n",
      "missing word: now-naked\n",
      "missing word: tee-shirts\n",
      "missing word: black-suited\n",
      "missing word: bad-asses\n",
      "missing word: eye-skewed\n",
      "missing word: mug-shot\n",
      "missing word: middle-background\n",
      "missing word: car-door\n",
      "missing word: whitens\n",
      "missing word: supered\n",
      "missing word: sweat-\n",
      "missing word: lunchpail\n",
      "missing word: accordioned\n",
      "missing word: palm-up\n",
      "missing word: gum-chewing\n",
      "missing word: mortices\n",
      "missing word: chroma-keyed\n",
      "missing word: five-\n",
      "missing word: floor-level\n",
      "missing word: driver's-window\n",
      "missing word: goggle-eyed\n",
      "missing word: someone.up\n",
      "missing word: someone.looks\n",
      "missing word: someone.as\n",
      "missing word: weeeeeeee\n",
      "missing word: whooooosh\n",
      "missing word: low-rider\n",
      "missing word: jack-rabbit\n",
      "missing word: slues\n",
      "missing word: thwap\n",
      "missing word: ear-splitting\n",
      "missing word: high-\n",
      "missing word: ba-weeeeeeee\n",
      "missing word: white-smocked\n",
      "missing word: inkpad\n",
      "missing word: bellower\n",
      "missing word: feet-the\n",
      "missing word: whipcracking\n",
      "missing word: whipcrack\n",
      "missing word: stock-still\n",
      "missing word: bottle-feeding\n",
      "missing word: thwok\n",
      "missing word: knee-level\n",
      "missing word: six-year-\n",
      "missing word: bam-shouting\n",
      "missing word: wing-\n",
      "missing word: v-area\n",
      "missing word: short-sleeve\n",
      "missing word: thwak\n",
      "missing word: someone.sits\n",
      "missing word: pimply-faced\n",
      "missing word: someone.from\n",
      "missing word: whoo-whoo\n",
      "missing word: semi-reflective\n",
      "missing word: unstraps\n",
      "missing word: smack-crack\n",
      "missing word: half-kneeling\n",
      "missing word: 7-\n",
      "missing word: light-and-siren\n",
      "missing word: pthunk\n",
      "missing word: ping-ping\n",
      "missing word: ka-chick\n",
      "missing word: ka-chock\n",
      "missing word: steensma\n",
      "missing word: platform-cubicle\n",
      "missing word: thoonk\n",
      "missing word: avoid-impact\n",
      "missing word: burning-inside\n",
      "missing word: mail-and-chained\n",
      "missing word: source-either\n",
      "missing word: middle-\n",
      "missing word: together-holding\n",
      "missing word: whoosh-someone\n",
      "missing word: late-middle-aged\n",
      "missing word: plastic-covered\n",
      "missing word: someone.in\n",
      "missing word: townlet\n",
      "missing word: locked-down\n",
      "missing word: ka-pop\n",
      "missing word: ka-boom\n",
      "missing word: sluing\n",
      "missing word: someone.to\n",
      "missing word: thunks\n",
      "missing word: shirtfront\n",
      "missing word: squeeze-lever\n",
      "missing word: burned-in\n",
      "missing word: someone.into\n",
      "missing word: squeeze-me\n",
      "missing word: stiff-arming\n",
      "missing word: rosy-cheeked\n",
      "missing word: weather-worn\n",
      "missing word: unhandsome\n",
      "missing word: dark-\n",
      "missing word: fore-ground\n",
      "missing word: powerful-looking\n",
      "missing word: copper-\n",
      "missing word: unpin\n",
      "missing word: flat-chested\n",
      "missing word: unattractively\n",
      "missing word: kitchen-lining\n",
      "missing word: blue-pin-striped\n",
      "missing word: semi-closeup\n",
      "missing word: divan-\n",
      "missing word: gesticulates\n",
      "missing word: half-opened\n",
      "missing word: palmful\n",
      "missing word: house-robe\n",
      "missing word: coat-stand\n",
      "missing word: long-focus\n",
      "missing word: view-finder\n",
      "missing word: half-turns\n",
      "missing word: drainboard\n",
      "missing word: newspaper-\n",
      "missing word: peroxided\n",
      "missing word: siffleuse\n",
      "missing word: newly-weds\n",
      "missing word: windowseat\n",
      "missing word: rope-tied\n",
      "missing word: abstractedly\n",
      "missing word: intelligent-appearing\n",
      "missing word: lip-stick\n",
      "missing word: hair-do\n",
      "missing word: canap\n",
      "missing word: flowing-haired\n",
      "missing word: black-lighted\n",
      "missing word: 'attache\n",
      "missing word: song-writers\n",
      "missing word: snifters\n",
      "missing word: poises\n",
      "missing word: over-feminine\n",
      "missing word: fire-escape\n",
      "missing word: long-\n",
      "missing word: black-covered\n",
      "missing word: 2-7099\n",
      "missing word: song-\n",
      "missing word: gun-shot\n",
      "missing word: red-lighted\n",
      "missing word: shitless\n",
      "missing word: crickett\n",
      "missing word: sweet-looking\n",
      "missing word: black-marble\n",
      "missing word: rec-room/basement\n",
      "missing word: leg-restraints\n",
      "missing word: over-enunciated\n",
      "missing word: spastically\n",
      "missing word: mini-fridge\n",
      "missing word: greek-lettered\n",
      "missing word: cheat-sheet\n",
      "missing word: all-but-deserted\n",
      "missing word: disinterestedly\n",
      "missing word: bottle-blonde\n",
      "missing word: seven-\n",
      "missing word: jack-hammer\n",
      "missing word: cat-scan\n",
      "missing word: strung-out\n",
      "missing word: charred-bottom\n",
      "missing word: brown-spotted\n",
      "missing word: co-\n",
      "missing word: elves-someone\n",
      "missing word: cirdan\n",
      "missing word: men.as\n",
      "missing word: holding-close\n",
      "missing word: earth.moving\n",
      "missing word: name.mordor\n",
      "missing word: map.closing\n",
      "missing word: dagorlad\n",
      "missing word: archers.orcs\n",
      "missing word: alliance.someone\n",
      "missing word: sword.narsil\n",
      "missing word: saruron\n",
      "missing word: charge.the\n",
      "missing word: ofsauron\n",
      "missing word: narsil.the\n",
      "missing word: gone.vaporized\n",
      "missing word: ring.transfixed\n",
      "missing word: field.isildur\n",
      "missing word: woods.the\n",
      "missing word: orcs.isildur\n",
      "missing word: water.as\n",
      "missing word: face.he\n",
      "missing word: swirls.a\n",
      "missing word: down.grasping\n",
      "missing word: still.the\n",
      "missing word: cavern.a\n",
      "missing word: water.in\n",
      "missing word: keeper.the\n",
      "missing word: rock.rising\n",
      "missing word: up.looking\n",
      "missing word: someone.his\n",
      "missing word: shire.60\n",
      "missing word: moment.then\n",
      "missing word: inn.towards\n",
      "missing word: scorcs\n",
      "missing word: hobbiton.a\n",
      "missing word: ignorcs\n",
      "missing word: end.a\n",
      "missing word: door.a\n",
      "missing word: indertiminate\n",
      "missing word: end.cozy\n",
      "missing word: around.enjoying\n",
      "missing word: end.he\n",
      "missing word: corner.it\n",
      "missing word: pocket.gripping\n",
      "missing word: branches.evaporating\n",
      "missing word: ale.someone\n",
      "missing word: water.with\n",
      "missing word: stool.he\n",
      "missing word: someone.a\n",
      "missing word: someone.he\n",
      "missing word: ring.he\n",
      "missing word: ring.glinting\n",
      "missing word: floor.someone\n",
      "missing word: pipe.staring\n",
      "missing word: turns.his\n",
      "missing word: envelope.someone\n",
      "missing word: gandolf\n",
      "missing word: door.leaving\n",
      "missing word: in.the\n",
      "missing word: ring.beneath\n",
      "missing word: barad-dur\n",
      "missing word: sky.casting\n",
      "missing word: tirith.he\n",
      "missing word: citidel\n",
      "missing word: nostrils.as\n",
      "missing word: doorway.fang\n",
      "missing word: quiet.someone\n",
      "missing word: envelope.revealing\n",
      "missing word: tongs.he\n",
      "missing word: hand.he\n",
      "missing word: red.as\n",
      "missing word: within.emanates\n",
      "missing word: riders.a\n",
      "missing word: knapsack.someone\n",
      "missing word: room.someone\n",
      "missing word: hobbiton.shrouded\n",
      "missing word: horse.people\n",
      "missing word: nods.he\n",
      "missing word: countryside.wading\n",
      "missing word: short.taking\n",
      "missing word: isengard.a\n",
      "missing word: ring-wall\n",
      "missing word: orthanc.hewn\n",
      "missing word: obsidian.rises\n",
      "missing word: door.horrified\n",
      "missing word: wall.pinned\n",
      "missing word: someone.blasting\n",
      "missing word: grasp.it\n",
      "missing word: chamber.as\n",
      "missing word: up.someone\n",
      "missing word: hedgerow.angry\n",
      "missing word: away.with\n",
      "missing word: ringwrath\n",
      "missing word: view.hooded\n",
      "missing word: ringwraith\n",
      "missing word: place.he\n",
      "missing word: hands.his\n",
      "missing word: slumps.as\n",
      "missing word: trees.slipping\n",
      "missing word: intently.someone\n",
      "missing word: friend.realizing\n",
      "missing word: river.and\n",
      "missing word: bree.a\n",
      "missing word: gatehouse.wild\n",
      "missing word: village.a\n",
      "missing word: bree.the\n",
      "missing word: them.lights\n",
      "missing word: lantern.a\n",
      "missing word: gatekeeper.as\n",
      "missing word: open.the\n",
      "missing word: ringwraits\n",
      "missing word: pillowis\n",
      "missing word: unstrap\n",
      "missing word: someone.lying\n",
      "missing word: night.the\n",
      "missing word: killed.as\n",
      "missing word: weathertop.they\n",
      "missing word: stider\n",
      "missing word: someone.eyes\n",
      "missing word: weathertop\n",
      "missing word: appear.brandishing\n",
      "missing word: leader.the\n",
      "missing word: shatters.the\n",
      "missing word: world.he\n",
      "missing word: athelas\n",
      "missing word: impression.a\n",
      "missing word: mud-splattered\n",
      "missing word: someone.we\n",
      "missing word: alsforth\n",
      "missing word: asfaloth\n",
      "missing word: trees.and\n",
      "missing word: bobbing.trees\n",
      "missing word: by.moonlight\n",
      "missing word: trees.the\n",
      "missing word: directions.a\n",
      "missing word: teeth.urges\n",
      "missing word: flares.someone\n",
      "missing word: window.dappled\n",
      "missing word: questioningly\n",
      "missing word: isengard.on\n",
      "missing word: gwaihir\n",
      "missing word: page.there\n",
      "missing word: pack.pots\n",
      "missing word: shadows.as\n",
      "missing word: wall.depicting\n",
      "missing word: narsil\n",
      "missing word: cloth-covered\n",
      "missing word: presence.he\n",
      "missing word: bridge.the\n",
      "missing word: evenstar\n",
      "missing word: free-peoples\n",
      "missing word: ring.unharmed\n",
      "missing word: motion.the\n",
      "missing word: ring.the\n",
      "missing word: screen.streams\n",
      "missing word: someone.breathing\n",
      "missing word: stands.he\n",
      "missing word: silence.someone\n",
      "missing word: scabbard.its\n",
      "missing word: someone.for\n",
      "missing word: shocked.the\n",
      "missing word: falters.his\n",
      "missing word: waters.the\n",
      "missing word: shale.as\n",
      "missing word: chain.he\n",
      "missing word: unhands\n",
      "missing word: birthsacks.flying\n",
      "missing word: caradhras\n",
      "missing word: snow.he\n",
      "missing word: lake.directly\n",
      "missing word: moria.a\n",
      "missing word: battle.the\n",
      "missing word: tentacleholding\n",
      "missing word: safety.someone\n",
      "missing word: tentacle.someone\n",
      "missing word: despoiled.dwarf\n",
      "missing word: passages.each\n",
      "missing word: waiting.while\n",
      "missing word: nervous.he\n",
      "missing word: nervously.he\n",
      "missing word: sword.and\n",
      "missing word: someone.who\n",
      "missing word: blow.suddenly\n",
      "missing word: wall.a\n",
      "missing word: over.someone\n",
      "missing word: down.people\n",
      "missing word: him.someone\n",
      "missing word: arrow.and\n",
      "missing word: coughs.takes\n",
      "missing word: dwarrowdelf\n",
      "missing word: door.as\n",
      "missing word: beast.melting\n",
      "missing word: hall.the\n",
      "missing word: shakes.an\n",
      "missing word: hesitates.someone\n",
      "missing word: stairway.someone\n",
      "missing word: someone.leaping\n",
      "missing word: pit.someone\n",
      "missing word: stone.without\n",
      "missing word: rail.at\n",
      "missing word: bridge.but\n",
      "missing word: last.pauses\n",
      "missing word: span.he\n",
      "missing word: balrog.staff\n",
      "missing word: hand.sword\n",
      "missing word: man-beast\n",
      "missing word: blade.like\n",
      "missing word: turns.a\n",
      "missing word: light.in\n",
      "missing word: seen.lothlorien\n",
      "missing word: flowers.above\n",
      "missing word: pillars.the\n",
      "missing word: mallorn\n",
      "missing word: others.nestled\n",
      "missing word: mallorns\n",
      "missing word: sun.green\n",
      "missing word: lothlorien\n",
      "missing word: smile.betraying\n",
      "missing word: open.as\n",
      "missing word: her.as\n",
      "missing word: jug.a\n",
      "missing word: pedestal.he\n",
      "missing word: intensely.someone\n",
      "missing word: down.in\n",
      "missing word: laughs.a\n",
      "missing word: elf-woman\n",
      "missing word: hand.his\n",
      "missing word: uruk-\n",
      "missing word: paint.a\n",
      "missing word: ceremony.the\n",
      "missing word: armored.breastplate.leg\n",
      "missing word: helmet.a\n",
      "missing word: isengard.they\n",
      "missing word: silverlode\n",
      "missing word: someone.strangely\n",
      "missing word: river.carved\n",
      "missing word: gondorian\n",
      "missing word: asleep.someone\n",
      "missing word: start.someone\n",
      "missing word: round.his\n",
      "missing word: trees.lost\n",
      "missing word: slowly.he\n",
      "missing word: neck.and\n",
      "missing word: steps.he\n",
      "missing word: seat.the\n",
      "missing word: hen.below\n",
      "missing word: someone.someone\n",
      "missing word: uncurls\n",
      "missing word: fist.in\n",
      "missing word: sun.someone\n",
      "missing word: trees.as\n",
      "missing word: sword.leaping\n",
      "missing word: falls.quickly\n",
      "missing word: tree.above\n",
      "missing word: path.while\n",
      "missing word: river.the\n",
      "missing word: lake.someone\n",
      "missing word: coughing.someone\n",
      "missing word: underwater.hands\n",
      "missing word: tears.the\n",
      "missing word: water.someone\n",
      "missing word: chest.his\n",
      "missing word: rauros\n",
      "missing word: affection.despite\n",
      "missing word: deagol\n",
      "missing word: deagol.strangling\n",
      "missing word: lembas\n",
      "missing word: foest\n",
      "missing word: .people\n",
      "missing word: edoras\n",
      "missing word: is-laid\n",
      "missing word: floorshow\n",
      "missing word: hate-\n",
      "missing word: pipe-weed\n",
      "missing word: islidur\n",
      "missing word: elven-smiths\n",
      "missing word: mindolluin\n",
      "missing word: outthrust\n",
      "missing word: ecthelion\n",
      "missing word: gondorians\n",
      "missing word: mumakil\n",
      "missing word: battle-scarred\n",
      "missing word: nearvertical\n",
      "missing word: rock-face\n",
      "missing word: razor-like\n",
      "missing word: uprushing\n",
      "missing word: murderous-looking\n",
      "missing word: snowmane\n",
      "missing word: dunharrow\n",
      "missing word: nazgul\n",
      "missing word: madril\n",
      "missing word: lowlying\n",
      "missing word: rag-dolls\n",
      "missing word: fell-beasts\n",
      "missing word: iorlas\n",
      "missing word: out-stretched\n",
      "missing word: reparied\n",
      "missing word: camp-fires\n",
      "missing word: unsaddling\n",
      "missing word: whinnies\n",
      "missing word: gamling\n",
      "missing word: ashen-faced\n",
      "missing word: dimholt\n",
      "missing word: re-sheathes\n",
      "missing word: brego\n",
      "missing word: dwimorberg\n",
      "missing word: firneedles\n",
      "missing word: anduril\n",
      "missing word: hollow-eyed\n",
      "missing word: thwat\n",
      "missing word: side-steps\n",
      "missing word: wolf.this\n",
      "missing word: mountainstone\n",
      "missing word: bare-foot\n",
      "missing word: booomm\n",
      "missing word: tower.shagrat\n",
      "missing word: oil-soaked\n",
      "missing word: elephant-like\n",
      "missing word: war-tower\n",
      "missing word: witch-someone\n",
      "missing word: fell-beast\n",
      "missing word: jaws.\n",
      "missing word: blood-lust\n",
      "missing word: witchking\n",
      "missing word: war-towers\n",
      "missing word: uruks\n",
      "missing word: foul-looking\n",
      "missing word: half-bowed\n",
      "missing word: .straight\n",
      "missing word: hobbit-face\n",
      "missing word: sword-raised\n",
      "missing word: ash-\n",
      "missing word: fume-laden\n",
      "missing word: sammath\n",
      "missing word: imploringly\n",
      "missing word: arm-in-arm\n",
      "missing word: boat-house\n",
      "missing word: toddles\n",
      "missing word: hand-addressed\n",
      "missing word: half-unpacked\n",
      "missing word: half-stripped\n",
      "missing word: endtable\n",
      "missing word: face-forward\n",
      "missing word: i.v\n",
      "missing word: half-octave\n",
      "missing word: big-top\n",
      "missing word: stilt-walking\n",
      "missing word: firebreathers\n",
      "missing word: owner-and-ringmaster\n",
      "missing word: mid-twirl\n",
      "missing word: antsier\n",
      "missing word: still-spry\n",
      "missing word: half-laughs\n",
      "missing word: sky-written\n",
      "missing word: just-woken\n",
      "missing word: football-\n",
      "missing word: skynyrd-loving\n",
      "missing word: fraternity-proud\n",
      "missing word: ass-whupping\n",
      "missing word: unshittable\n",
      "missing word: official-looking\n",
      "missing word: rifle-toting\n",
      "missing word: frankensteinian\n",
      "missing word: roll-top\n",
      "missing word: re-reads\n",
      "missing word: half-hung\n",
      "missing word: half-smiles\n",
      "missing word: half-there\n",
      "missing word: unscrews\n",
      "missing word: black-and\n",
      "missing word: outflung\n",
      "missing word: blubbers\n",
      "missing word: boomy\n",
      "missing word: strike-scoring\n",
      "missing word: sunglassed\n",
      "missing word: sixtyish\n",
      "missing word: beat-\n",
      "missing word: kalhua\n",
      "missing word: high-ceilinged\n",
      "missing word: teartracks\n",
      "missing word: dacron-polyester\n",
      "missing word: beer-swilling\n",
      "missing word: black-and-red\n",
      "missing word: paint-spattered\n",
      "missing word: thumbhole\n",
      "missing word: unclips\n",
      "missing word: paper-wrapped\n",
      "missing word: rat-tat-tat-tat\n",
      "missing word: clunks\n",
      "missing word: sallow-faced\n",
      "missing word: cover-alls\n",
      "missing word: freshly-mixed\n",
      "missing word: frog-marched\n",
      "missing word: catty-corner\n",
      "missing word: letter-sized\n",
      "missing word: untucks\n",
      "missing word: stuporously\n",
      "missing word: craggily\n",
      "missing word: slicked-back\n",
      "missing word: thin-lipped\n",
      "missing word: modishly\n",
      "missing word: fifteen-year-\n",
      "missing word: straightbacked\n",
      "missing word: blow-dried\n",
      "missing word: blanket-tossing\n",
      "missing word: fiftyish\n",
      "missing word: treehorn\n",
      "missing word: mini-\n",
      "missing word: busby-berkley\n",
      "missing word: torpedolike\n",
      "missing word: barrel-roll\n",
      "missing word: bloo-whup\n",
      "missing word: dreds\n",
      "missing word: corn-fed\n",
      "missing word: snow-swept\n",
      "missing word: checkpad\n",
      "missing word: teutonically\n",
      "missing word: camera-side\n",
      "missing word: bandage-\n",
      "missing word: hip-grinding\n",
      "missing word: wind-swept\n",
      "missing word: sticklike\n",
      "missing word: new-cut\n",
      "missing word: franknum\n",
      "missing word: reapplies\n",
      "missing word: desk-light\n",
      "missing word: frog-\n",
      "missing word: brothel-cum-club\n",
      "missing word: cafe-bar\n",
      "missing word: gout-ridden\n",
      "missing word: suspender-belts\n",
      "missing word: ladder-staircase\n",
      "missing word: expressionlessly\n",
      "missing word: braddocks\n",
      "missing word: readdressing\n",
      "missing word: grinningly\n",
      "missing word: twenty-\n",
      "missing word: searchingly\n",
      "missing word: t-\n",
      "missing word: elvator\n",
      "missing word: half-packed\n",
      "missing word: exhausted-looking\n",
      "missing word: skee-ball\n",
      "missing word: mid-progress\n",
      "missing word: twelve-thirty\n",
      "missing word: buzzard-like\n",
      "missing word: racker\n",
      "missing word: ivory-pointed\n",
      "missing word: duelers\n",
      "missing word: one-thirty\n",
      "missing word: discomposure\n",
      "missing word: losings\n",
      "missing word: bare-\n",
      "missing word: thick-fleshed\n",
      "missing word: obscene-looking\n",
      "missing word: ground-glass\n",
      "missing word: half-sleep\n",
      "missing word: distrustfully\n",
      "missing word: kentucky-bound\n",
      "missing word: cork-tipped\n",
      "missing word: wood-paneled\n",
      "missing word: plainclothesman\n",
      "missing word: three-quarter-\n",
      "missing word: kerbing\n",
      "missing word: tsk-tsks\n",
      "missing word: jiggerful\n",
      "missing word: round-faced\n",
      "missing word: land-office\n",
      "missing word: over-vivacious\n",
      "missing word: shushings\n",
      "missing word: cuspidor\n",
      "missing word: fore-\n",
      "missing word: coat-rack\n",
      "missing word: brief-case\n",
      "missing word: tooth-picks\n",
      "missing word: two-wash-basin\n",
      "missing word: fifty-cent\n",
      "missing word: checkroom\n",
      "missing word: half-crazed\n",
      "missing word: hairdress\n",
      "missing word: wheel-chairs\n",
      "missing word: terry-cloth\n",
      "missing word: peep-hole\n",
      "missing word: wheel-chair\n",
      "missing word: d.t\n",
      "missing word: ante-room\n",
      "missing word: half-stupor\n",
      "missing word: door-\n",
      "missing word: writing.but\n",
      "missing word: papooses\n",
      "missing word: hudred\n",
      "missing word: six-fingered\n",
      "missing word: turret-shaped\n",
      "missing word: black-green\n",
      "missing word: r.o.u.s\n",
      "missing word: r.o.u.s.\n",
      "missing word: off-\n",
      "missing word: quick-eyed\n",
      "missing word: alarge\n",
      "missing word: puffy-eyed\n",
      "missing word: beautific\n",
      "missing word: flickeringly\n",
      "missing word: night-dresses\n",
      "missing word: forte-\n",
      "missing word: chamber-pot\n",
      "missing word: still-falling\n",
      "missing word: half-conscious\n",
      "missing word: frock-coat\n",
      "missing word: forte-piano\n",
      "missing word: heavy-\n",
      "missing word: self-approving\n",
      "missing word: elaborate-looking\n",
      "missing word: turned-cut\n",
      "missing word: exotic-looking\n",
      "missing word: wind-musicians\n",
      "missing word: time-serving\n",
      "missing word: side-table\n",
      "missing word: inkstands\n",
      "missing word: andrai\n",
      "missing word: olivewood\n",
      "missing word: martern\n",
      "missing word: arten\n",
      "missing word: entschlossen\n",
      "missing word: over-excited\n",
      "missing word: curtsies\n",
      "missing word: once-handsome\n",
      "missing word: signet-ring\n",
      "missing word: inquiringly\n",
      "missing word: coarse-looking\n",
      "missing word: low-ceilinged\n",
      "missing word: travel-stained\n",
      "missing word: contenti\n",
      "missing word: saremo\n",
      "missing word: 'holy\n",
      "missing word: inkstand\n",
      "missing word: orsini-\n",
      "missing word: recitativo\n",
      "missing word: confutatis\n",
      "missing word: postillions\n",
      "missing word: guttered\n",
      "missing word: toe-searches\n",
      "missing word: hair-combing\n",
      "missing word: oil-lease\n",
      "missing word: hot-breath\n",
      "missing word: money-filled\n",
      "missing word: self-angry\n",
      "missing word: resignment\n",
      "missing word: undiscernible\n",
      "missing word: rain-sheeted\n",
      "missing word: pushbell\n",
      "missing word: soft-\n",
      "missing word: lock-folds\n",
      "missing word: napkin-covered\n",
      "missing word: halfslip\n",
      "missing word: rehangs\n",
      "missing word: by-\n",
      "missing word: shower-bar\n",
      "missing word: closed-for-lunch\n",
      "missing word: bird-ridden\n",
      "missing word: one-lamp\n",
      "missing word: shrike-\n",
      "missing word: desk-counter\n",
      "missing word: relaxedly\n",
      "missing word: uncanoped\n",
      "missing word: chiffonier\n",
      "missing word: big-doored\n",
      "missing word: full-\n",
      "missing word: pier-glass\n",
      "missing word: recamier\n",
      "missing word: clock-wise\n",
      "missing word: leathery-brown\n",
      "missing word: high-neck\n",
      "missing word: well-pressed\n",
      "missing word: hand-ironed\n",
      "missing word: near-\n",
      "missing word: wire-covered\n",
      "missing word: reprovingly\n",
      "missing word: no-whereness\n",
      "missing word: parchment-type\n",
      "missing word: instanly\n",
      "missing word: dimly-lit\n",
      "missing word: blood-chilling\n",
      "missing word: get-well\n",
      "missing word: statue-like\n",
      "missing word: eight-year-olds\n",
      "missing word: over-confidence\n",
      "missing word: over-moussed\n",
      "missing word: yoohoo\n",
      "missing word: ident-a-kit\n",
      "missing word: book-\n",
      "missing word: barely-managing-\n",
      "missing word: to-\n",
      "missing word: black/anti-white\n",
      "missing word: crowd-is\n",
      "missing word: ruffled-haired\n",
      "missing word: half-smile\n",
      "missing word: c'mere\n",
      "missing word: aw-\n",
      "missing word: way-to-passionate-to-mean-anything-else\n",
      "missing word: ear-to-ear\n",
      "missing word: way-too-much\n",
      "missing word: 'just\n",
      "missing word: blanket-covered\n",
      "missing word: 'idiosyncratic\n",
      "missing word: 'bluntman\n",
      "missing word: mimicing\n",
      "missing word: 'shared\n",
      "missing word: 'hopefully\n",
      "missing word: 'good\n",
      "missing word: 'bye\n",
      "missing word: fan-boys\n",
      "missing word: book-and-photograph-filled\n",
      "missing word: memo-cluttered\n",
      "missing word: on-and-offscreen\n",
      "missing word: plastic-enclosed\n",
      "missing word: fast-tracking\n",
      "missing word: sun-filled\n",
      "missing word: examining-room\n",
      "missing word: otoscope\n",
      "missing word: sonometer\n",
      "missing word: pot-and-pan-\n",
      "missing word: paint-splattered\n",
      "missing word: shelf-\n",
      "missing word: hearing-test\n",
      "missing word: slouched-\n",
      "missing word: electronystagmography\n",
      "missing word: bser\n",
      "missing word: cranelike\n",
      "missing word: high-backed\n",
      "missing word: half-eaten\n",
      "missing word: red-imprinted\n",
      "missing word: estlin\n",
      "missing word: science-\n",
      "missing word: unbelting\n",
      "missing word: sleazy-looking\n",
      "missing word: swills\n",
      "missing word: stricken-\n",
      "missing word: evening-lit\n",
      "missing word: open-\n",
      "missing word: air-displayed\n",
      "missing word: coatcheck\n",
      "missing word: book-laden\n",
      "missing word: dishtowel\n",
      "missing word: food-laden\n",
      "missing word: inside.and\n",
      "missing word: almost-transparent\n",
      "missing word: glass-doored\n",
      "missing word: pick-axes\n",
      "missing word: olive-colored\n",
      "missing word: canvas-covered\n",
      "missing word: shirtsleeve\n",
      "missing word: thirty-foot\n",
      "missing word: cloth-wrapped\n",
      "missing word: museum-quality\n",
      "missing word: book-lined\n",
      "missing word: oil-slick\n",
      "missing word: knee-\n",
      "missing word: rat-infested\n",
      "missing word: postcard-perfect\n",
      "missing word: matchwood\n",
      "missing word: water-soaked\n",
      "missing word: book-burning\n",
      "missing word: bi-plane\n",
      "missing word: messerschmidts\n",
      "missing word: ratta-a-tat-tat\n",
      "missing word: under-fuselage\n",
      "missing word: hate-staring\n",
      "missing word: fast-approaching\n",
      "missing word: mid-swipe\n",
      "missing word: over-awed\n",
      "missing word: snouth\n",
      "missing word: snail-like\n",
      "missing word: neuralyzer\n",
      "missing word: twelve-gauge\n",
      "missing word: 4-eyes\n",
      "missing word: ten-fingered\n",
      "missing word: arquillian\n",
      "missing word: gearshifts\n",
      "missing word: baltian\n",
      "missing word: mean-looking\n",
      "missing word: passerbys\n",
      "missing word: deatomizer\n",
      "missing word: real-live\n",
      "missing word: wirier\n",
      "missing word: looooong\n",
      "missing word: unsnaps\n",
      "missing word: wicked-looking\n",
      "missing word: triple-barreled\n",
      "missing word: reloader\n",
      "missing word: up.revealing\n",
      "missing word: triumph.he\n",
      "missing word: 'wait\n",
      "missing word: beweildered\n",
      "missing word: un-\n",
      "missing word: going-on\n",
      "missing word: embarrassedly\n",
      "missing word: cogitates\n",
      "missing word: offcenter\n",
      "missing word: uten-\n",
      "missing word: ob-\n",
      "missing word: viously\n",
      "missing word: fascinat-\n",
      "missing word: pas-\n",
      "missing word: 'observer\n",
      "missing word: mid-\n",
      "missing word: sur-\n",
      "missing word: includ-\n",
      "missing word: post-mortems\n",
      "missing word: aquiver\n",
      "missing word: cogitating\n",
      "missing word: speaker-\n",
      "missing word: folded-up\n",
      "missing word: side-stick\n",
      "missing word: unbuckles\n",
      "missing word: still-unconscious\n",
      "missing word: broken-off\n",
      "missing word: groggy-eyed\n",
      "missing word: semi-slumped\n",
      "missing word: over-stuffed\n",
      "missing word: re-hangs\n",
      "missing word: apparate\n",
      "missing word: death-eater\n",
      "missing word: borgin\n",
      "missing word: thin-faced\n",
      "missing word: rose-tinted\n",
      "missing word: spectrespecs\n",
      "missing word: quibbler\n",
      "missing word: slytherins\n",
      "missing word: sopophorous\n",
      "missing word: linedinfront\n",
      "missing word: triallists\n",
      "missing word: quaffle\n",
      "missing word: butterbeer\n",
      "missing word: torwards\n",
      "missing word: bludger\n",
      "missing word: raven-haired\n",
      "missing word: ankle-deep\n",
      "missing word: spyhole\n",
      "missing word: prises\n",
      "missing word: cabine\n",
      "missing word: shoked\n",
      "missing word: lead-paned\n",
      "missing word: vine-like\n",
      "missing word: spume\n",
      "missing word: previously-\n",
      "missing word: auror\n",
      "missing word: skull-\n",
      "missing word: apair\n",
      "missing word: high-back\n",
      "missing word: intricately-carved\n",
      "missing word: snaffles\n",
      "missing word: hawkishly\n",
      "missing word: urine-colored\n",
      "missing word: be-suited\n",
      "missing word: tee-\n",
      "missing word: eye-glass\n",
      "missing word: doe-eyed\n",
      "missing word: white-\n",
      "missing word: a-ok\n",
      "missing word: stony-faced\n",
      "missing word: sheepish-looking\n",
      "missing word: plonks\n",
      "missing word: non-plussed\n",
      "missing word: half-light\n",
      "missing word: brown-suited\n",
      "missing word: fry-up\n",
      "missing word: scrabbles\n",
      "missing word: custome\n",
      "missing word: goblet-like\n",
      "missing word: pin-striped\n",
      "missing word: jives\n",
      "missing word: brill-creamed\n",
      "missing word: love-\n",
      "missing word: mop-head\n",
      "missing word: flame-haired\n",
      "missing word: liquorish\n",
      "missing word: hamburger-shaped\n",
      "missing word: crumbing\n",
      "missing word: double-chinned\n",
      "missing word: hightails\n",
      "missing word: double-fronted\n",
      "missing word: lorings\n",
      "missing word: overskirt\n",
      "missing word: elasticated\n",
      "missing word: mustachio\n",
      "missing word: six-years-old\n",
      "missing word: glassily\n",
      "missing word: falteringly\n",
      "missing word: sufferingly\n",
      "missing word: grey-painted\n",
      "missing word: neatly-arranged\n",
      "missing word: wire-framed\n",
      "missing word: smooth-skinned\n",
      "missing word: high-five\n",
      "missing word: semi-\n",
      "missing word: tuk-tuks\n",
      "missing word: latikas\n",
      "missing word: tuk-tuk\n",
      "missing word: five-fifteen\n",
      "missing word: cubiclal\n",
      "missing word: watche\n",
      "missing word: pasty-faced\n",
      "missing word: grim-faced\n",
      "missing word: primps\n",
      "missing word: calfland\n",
      "missing word: gawps\n",
      "missing word: un-sure\n",
      "missing word: four-dour\n",
      "missing word: combers\n",
      "missing word: ploddeds\n",
      "missing word: brazened\n",
      "missing word: cunks\n",
      "missing word: eye-hole\n",
      "missing word: gray-head\n",
      "missing word: snug-fitting\n",
      "missing word: high-colored\n",
      "missing word: neck-craned\n",
      "missing word: ones-over\n",
      "missing word: red-head\n",
      "missing word: wood-carved\n",
      "missing word: cable-car\n",
      "missing word: multi-bladed\n",
      "missing word: grecian-style\n",
      "missing word: bored-looking\n",
      "missing word: columnated\n",
      "missing word: steam-filled\n",
      "missing word: mid-swing\n",
      "missing word: happents\n",
      "missing word: understandingly\n",
      "missing word: anxious-looking\n",
      "missing word: short-circuits\n",
      "missing word: octavious\n",
      "missing word: triveil\n",
      "missing word: hree\n",
      "missing word: spear-like\n",
      "missing word: spider-\n",
      "missing word: canope\n",
      "missing word: mancheses\n",
      "missing word: terrace-like\n",
      "missing word: judders\n",
      "missing word: dark-rimmed\n",
      "missing word: acroos\n",
      "missing word: limbos\n",
      "missing word: cambers\n",
      "missing word: tear-filled\n",
      "missing word: -man\n",
      "missing word: about-turns\n",
      "missing word: ever-\n",
      "missing word: off-the-shoulder\n",
      "missing word: posey-style\n",
      "missing word: reproachfully\n",
      "missing word: hammock-like\n",
      "missing word: green-and-black-\n",
      "missing word: bronwhead\n",
      "missing word: sand-like\n",
      "missing word: paw-like\n",
      "missing word: blonde-head\n",
      "missing word: nodds\n",
      "missing word: silver-head\n",
      "missing word: doddery\n",
      "missing word: lavishly-furnished\n",
      "missing word: stoney-faced\n",
      "missing word: tarpaulin-\n",
      "missing word: unfastens\n",
      "missing word: tightly-packed\n",
      "missing word: high-fiving\n",
      "missing word: chivalrously\n",
      "missing word: floppy-head\n",
      "missing word: hip-thrusting\n",
      "missing word: black-\n",
      "missing word: pretz\n",
      "missing word: scocks\n",
      "missing word: encaged\n",
      "missing word: kong-like\n",
      "missing word: half-built\n",
      "missing word: half-disintegrate\n",
      "missing word: half-disintegrates\n",
      "missing word: submersives\n",
      "missing word: gangplanks\n",
      "missing word: two-manned\n",
      "missing word: zig-zaging\n",
      "missing word: ruby-colored\n",
      "missing word: unlaces\n",
      "missing word: blank-faced\n",
      "missing word: open-mouth\n",
      "missing word: fully-clothed\n",
      "missing word: steamed-up\n",
      "missing word: thrugh\n",
      "missing word: inch-\n",
      "missing word: chartroom\n",
      "missing word: pasengers\n",
      "missing word: band-leader\n",
      "missing word: auburn-haired\n",
      "missing word: -deep\n",
      "missing word: third-\n",
      "missing word: shoulder-high\n",
      "missing word: eaching\n",
      "missing word: wear.sits\n",
      "missing word: unclasps\n",
      "missing word: curly-haired\n",
      "missing word: somber-looking\n",
      "missing word: well-cushioned\n",
      "missing word: moon-faced\n",
      "missing word: stilettoed\n",
      "missing word: playsuit\n",
      "missing word: ringleted\n",
      "missing word: doom-laden\n",
      "missing word: unpins\n",
      "missing word: pursuited\n",
      "missing word: bourbon-\n",
      "missing word: tipsily\n",
      "missing word: simpers\n",
      "missing word: josefo\n",
      "missing word: owl-like\n",
      "missing word: barely-disguised\n",
      "missing word: chocolate-colored\n",
      "missing word: elegantly-dressed\n",
      "missing word: footes\n",
      "missing word: full-white\n",
      "missing word: body-hugging\n",
      "missing word: glugs\n",
      "missing word: off-the-\n",
      "missing word: flirtively\n",
      "missing word: chlidren\n",
      "missing word: half-beast\n",
      "missing word: re-wraps\n",
      "missing word: holdall\n",
      "missing word: pityingly\n",
      "missing word: sliver-gey\n",
      "missing word: unhearing\n",
      "missing word: untwists\n",
      "missing word: unreacting\n",
      "missing word: open-neck\n",
      "missing word: freewheels\n",
      "missing word: zip-up\n",
      "missing word: disbelievingly\n",
      "missing word: whitout\n",
      "missing word: swithched\n",
      "missing word: garnd\n",
      "missing word: infornt\n",
      "missing word: long-hair\n",
      "missing word: nightfalls\n",
      "missing word: masiive\n",
      "missing word: curly-hair\n",
      "missing word: pwers\n",
      "missing word: sweage\n",
      "missing word: thorws\n",
      "missing word: accelarates\n",
      "missing word: fissur\n",
      "missing word: controlls\n",
      "missing word: rised\n",
      "missing word: charili\n",
      "missing word: streching\n",
      "missing word: awe-struck\n",
      "missing word: reserch\n",
      "missing word: breakes\n",
      "missing word: erruption\n",
      "missing word: arms-outstretched\n",
      "missing word: noahs\n",
      "missing word: antinov\n",
      "missing word: decends\n",
      "missing word: himslef\n",
      "missing word: opnes\n",
      "missing word: corgies\n",
      "missing word: nampan\n",
      "missing word: brigde\n",
      "missing word: lunchpad\n",
      "missing word: turch\n",
      "missing word: seald\n",
      "missing word: tanzen\n",
      "missing word: geards\n",
      "missing word: decending\n",
      "missing word: hundereds\n",
      "missing word: whipe\n",
      "missing word: re-man\n",
      "missing word: floded\n",
      "missing word: anti-chamber\n",
      "missing word: decend\n",
      "missing word: brookyln\n",
      "missing word: sunbathes\n",
      "missing word: moisturizes\n",
      "missing word: twirly\n",
      "missing word: high-stepping\n",
      "missing word: mini-dresses\n",
      "missing word: open-shirted\n",
      "missing word: mud-spattered\n",
      "missing word: bikini-wearing\n",
      "missing word: not-so-happy\n",
      "missing word: figure-\n",
      "missing word: snow-filled\n",
      "missing word: mooches\n",
      "missing word: well-toned\n",
      "missing word: not-so-\n",
      "missing word: not-so-great\n",
      "missing word: valeting\n",
      "missing word: balustrated\n",
      "missing word: un-flushing\n",
      "missing word: lustly\n",
      "missing word: skaterat\n",
      "missing word: skaterats\n",
      "missing word: goatee-bearded\n",
      "missing word: warmly.someone\n",
      "missing word: cotch\n",
      "missing word: hot-bodied\n",
      "missing word: masticates\n",
      "missing word: vainfully\n",
      "missing word: gift-wrapped\n",
      "missing word: safe-breaking\n",
      "missing word: pole-style\n",
      "missing word: fibracell\n",
      "missing word: semi-comatose\n",
      "missing word: over-refreshment\n",
      "missing word: occa\n",
      "missing word: side.the\n",
      "missing word: wierdly\n",
      "missing word: vechicles\n",
      "missing word: safe.someone\n",
      "missing word: suceeds\n",
      "missing word: window.someone\n",
      "missing word: shaterat\n",
      "missing word: cordoned-off\n",
      "missing word: four-by-four\n",
      "missing word: four-\n",
      "missing word: by-four\n",
      "missing word: four-by-fours\n",
      "missing word: noordermarkt\n",
      "missing word: dark-suited\n",
      "missing word: waste-ground\n",
      "missing word: rubbish-strewn\n",
      "missing word: spread-eagled\n",
      "missing word: nymphets\n",
      "missing word: apricot-colored\n",
      "missing word: once-over\n",
      "missing word: pixie-like\n",
      "missing word: hot-dog\n",
      "missing word: sun-bleached\n",
      "missing word: dantay-someone\n",
      "missing word: postie\n",
      "missing word: swaved\n",
      "missing word: sun-kissed\n",
      "missing word: bony-faced\n",
      "missing word: nose-nuzzles\n",
      "missing word: uncrosses\n",
      "missing word: cashmere-gloved\n",
      "missing word: melon-shaped\n",
      "missing word: lainley\n",
      "missing word: glugging\n",
      "missing word: champagne-glugging\n",
      "missing word: po-faced\n",
      "missing word: blind-folded\n",
      "missing word: solemn-faced\n",
      "missing word: 837.57\n",
      "missing word: salaciously\n",
      "missing word: ex-friend\n",
      "missing word: rain-slick\n",
      "missing word: peachy-colored\n",
      "missing word: photo-sized\n",
      "missing word: gob-smacked\n",
      "missing word: cut-glass\n",
      "missing word: comnidyne\n",
      "missing word: pellit\n",
      "missing word: comb-over\n",
      "missing word: troubled-looking\n",
      "missing word: sharp-suited\n",
      "missing word: on-suite\n",
      "missing word: hoody\n",
      "missing word: reorientates\n",
      "missing word: low-gas\n",
      "missing word: white-knuckling\n",
      "missing word: barbicide\n",
      "missing word: skiptracer\n",
      "missing word: dirt-covered\n",
      "missing word: approaches.someone\n",
      "missing word: hard-lines\n",
      "missing word: smoothered\n",
      "missing word: forwad\n",
      "missing word: double-take\n",
      "missing word: construction-filled\n",
      "missing word: uncuffs\n",
      "missing word: rear-ends\n",
      "missing word: someone-style\n",
      "missing word: conference-room\n",
      "missing word: throat-punches\n",
      "missing word: tases\n",
      "missing word: black-tied\n",
      "missing word: broad-bladed\n",
      "missing word: burlier\n",
      "missing word: 30'ish\n",
      "missing word: anomalous-growing\n",
      "missing word: thoughtful-looking\n",
      "missing word: punky-looking\n",
      "missing word: gouts\n",
      "missing word: grave-looking\n",
      "missing word: closely-cropped\n",
      "missing word: fleshy-jowled\n",
      "missing word: christified\n",
      "missing word: smartly-dressed\n",
      "missing word: urchin-like\n",
      "missing word: fauchelevant\n",
      "missing word: ratty-haired\n",
      "missing word: beggers\n",
      "missing word: thenardiers\n",
      "missing word: grovels\n",
      "missing word: grave-faced\n",
      "missing word: gavroch\n",
      "missing word: wheeping\n",
      "missing word: cossete\n",
      "missing word: over-bed\n",
      "missing word: rain-\n",
      "missing word: sellotaped\n",
      "missing word: stripy\n",
      "missing word: shaggy-haired\n",
      "missing word: mouth-watering\n",
      "missing word: squeezy\n",
      "missing word: spun-sugar\n",
      "missing word: square-faced\n",
      "missing word: berber-style\n",
      "missing word: cafetiere\n",
      "missing word: lamp-lit\n",
      "missing word: spoon-feeds\n",
      "missing word: scarlet-haired\n",
      "missing word: longbourn\n",
      "missing word: napoleonic-style\n",
      "missing word: empire-line\n",
      "missing word: do-si-do\n",
      "missing word: sozzled\n",
      "missing word: meryton\n",
      "missing word: proud-looking\n",
      "missing word: austere-looking\n",
      "missing word: half-dressed\n",
      "missing word: bingleys\n",
      "missing word: wooden-floored\n",
      "missing word: red-coated\n",
      "missing word: eye-contact\n",
      "missing word: frustratedly\n",
      "missing word: ivy-clad\n",
      "missing word: poker-faced\n",
      "missing word: overdress\n",
      "missing word: hunsford\n",
      "missing word: sandy-haired\n",
      "missing word: hatcheck\n",
      "missing word: trench-coated\n",
      "missing word: baby-faced\n",
      "missing word: three-propeller\n",
      "missing word: chevron-shaped\n",
      "missing word: bird-winged\n",
      "missing word: light-heartedly\n",
      "missing word: fur-collared\n",
      "missing word: thin-\n",
      "missing word: rubber-neck\n",
      "missing word: care-worn\n",
      "missing word: metro-goldman-mayer\n",
      "missing word: aston-martin\n",
      "missing word: steely-eyed\n",
      "missing word: alfa-romeo\n",
      "missing word: slow-mo\n",
      "missing word: bullet-stream\n",
      "missing word: bulllet\n",
      "missing word: siloute\n",
      "missing word: gloomed\n",
      "missing word: siloutte\n",
      "missing word: high-heels\n",
      "missing word: near-tired\n",
      "missing word: drab-tiled\n",
      "missing word: switch-blade\n",
      "missing word: head-butted\n",
      "missing word: finger-breaking\n",
      "missing word: werving\n",
      "missing word: two-boat\n",
      "missing word: populant\n",
      "missing word: retirement-aged\n",
      "missing word: kunslerangang\n",
      "missing word: white-shirted\n",
      "missing word: tousle-headed\n",
      "missing word: venches\n",
      "missing word: rown\n",
      "missing word: grooup\n",
      "missing word: wide-sleeved\n",
      "missing word: knuckleduster\n",
      "missing word: cleaned-up\n",
      "missing word: underground-railway\n",
      "missing word: hand-knitted\n",
      "missing word: two-colored\n",
      "missing word: nosegays\n",
      "missing word: shell-shaped\n",
      "missing word: cheekboned\n",
      "missing word: karate-chops\n",
      "missing word: back-flips\n",
      "missing word: block-and-tackle\n",
      "missing word: spruced-up\n",
      "missing word: decants\n",
      "missing word: paddle-steamer\n",
      "missing word: addrelle\n",
      "missing word: coat-of\n",
      "missing word: -arms\n",
      "missing word: worktops\n",
      "missing word: glass-paned\n",
      "missing word: worktables\n",
      "missing word: long-barreled\n",
      "missing word: rough-looking\n",
      "missing word: heavy-caliber\n",
      "missing word: needle-sharp\n",
      "missing word: uncaps\n",
      "missing word: chloroformed\n",
      "missing word: hyperemic\n",
      "missing word: especially-adapted\n",
      "missing word: pumped-up\n",
      "missing word: un-shed\n",
      "missing word: cinay\n",
      "missing word: grimed\n",
      "missing word: un-dead\n",
      "missing word: un-metaled\n",
      "missing word: skull-like\n",
      "missing word: hoovered\n",
      "missing word: star-sprinkled\n",
      "missing word: twiddles\n",
      "missing word: spaghetti-strap\n",
      "missing word: to-go\n",
      "missing word: wand-like\n",
      "missing word: co-room\n",
      "missing word: chain-locked\n",
      "missing word: mini-van\n",
      "missing word: bloody-\n",
      "missing word: gut-punches\n",
      "missing word: cross-\n",
      "missing word: impensively\n",
      "missing word: coldcocks\n",
      "missing word: murble\n",
      "missing word: green-carpeted\n",
      "missing word: white-spine\n",
      "missing word: hat-wearing\n",
      "missing word: half-closes\n",
      "missing word: wicker-backed\n",
      "missing word: straight-backed\n",
      "missing word: streetlamp\n",
      "missing word: body-warmer\n",
      "missing word: rain-washed\n",
      "missing word: fly-ridden\n",
      "missing word: blides\n",
      "missing word: swan-like\n",
      "missing word: -skelter\n",
      "missing word: bed-bound\n",
      "missing word: elaborately-carved\n",
      "missing word: bottomup\n",
      "missing word: autumnal-colored\n",
      "missing word: chubby-cheeked\n",
      "missing word: sallow-skinned\n",
      "missing word: white-clapboard\n",
      "missing word: gray-slate\n",
      "missing word: contented-looking\n",
      "missing word: half-covered\n",
      "missing word: spot-lit\n",
      "missing word: flood-lit\n",
      "missing word: fine-featured\n",
      "missing word: confetti-covered\n",
      "missing word: bright-yellow\n",
      "missing word: white-wall\n",
      "missing word: four-tiered\n",
      "missing word: grey-faced\n",
      "missing word: flower-filled\n",
      "missing word: dewy-eyed\n",
      "missing word: rainclouds\n",
      "missing word: orange-squeezing\n",
      "missing word: wild-haired\n",
      "missing word: hard-faced\n",
      "missing word: slow-dancing\n",
      "missing word: pale-pink\n",
      "missing word: deusenberg\n",
      "missing word: blood-spattered\n",
      "missing word: heavy-browed\n",
      "missing word: sun-shaped\n",
      "missing word: gentled\n",
      "missing word: curtseys\n",
      "missing word: anguised\n",
      "missing word: ansd\n",
      "missing word: ksxp\n",
      "missing word: snogs\n",
      "missing word: flosses\n",
      "missing word: weather-girl\n",
      "missing word: tea-break\n",
      "missing word: thick-set\n",
      "missing word: figure-hugging\n",
      "missing word: latex-gloved\n",
      "missing word: amber-tinted\n",
      "missing word: glum-faced\n",
      "missing word: yellow-tinted\n",
      "missing word: moppy\n",
      "missing word: low-lit\n",
      "missing word: card-reading\n",
      "missing word: length-\n",
      "missing word: cross-wise\n",
      "missing word: craggy-faced\n",
      "missing word: i-told-you-so\n",
      "missing word: alpha-tech\n",
      "missing word: lodge-\n",
      "missing word: arm-in\n",
      "missing word: -arm\n",
      "missing word: cheek-to-cheek\n",
      "missing word: pat-a-cake\n",
      "missing word: slush-covered\n",
      "missing word: arrival/departure\n",
      "missing word: pre-explosion\n",
      "missing word: four-by-\n",
      "missing word: red-carpeted\n",
      "missing word: flag-\n",
      "missing word: cropped-haired\n",
      "missing word: gooping\n",
      "missing word: videoing\n",
      "missing word: heighered\n",
      "missing word: heavy-featured\n",
      "missing word: pony-tailed\n",
      "missing word: hold-all\n",
      "missing word: chloroforms\n",
      "missing word: shiftily\n",
      "missing word: power-slides\n",
      "missing word: parsemage\n",
      "missing word: single-arched\n",
      "missing word: mirror-\n",
      "missing word: portrait-lined\n",
      "missing word: now-darkened\n",
      "missing word: bottle-green\n",
      "missing word: unshed\n",
      "missing word: curricle\n",
      "missing word: wickhams\n",
      "missing word: awkward-looking\n",
      "missing word: halll\n",
      "missing word: scrimming\n",
      "missing word: old-timbered\n",
      "missing word: sacre-coeur\n",
      "missing word: re-assuring\n",
      "missing word: red-bearded\n",
      "missing word: up-ends\n",
      "missing word: pipe-smoking\n",
      "missing word: star-dusted\n",
      "missing word: -hand\n",
      "missing word: crocodile-skinned\n",
      "missing word: sun-blinds\n",
      "missing word: wide-\n",
      "missing word: magic-man\n",
      "missing word: pell-mell\n",
      "missing word: black-shirts\n",
      "missing word: spotss\n",
      "missing word: broodily\n",
      "missing word: kissely\n",
      "missing word: nowogrodek\n",
      "missing word: low-tiled\n",
      "missing word: scwols\n",
      "missing word: naliboki\n",
      "missing word: oinment\n",
      "missing word: frost-bitten\n",
      "missing word: stakiewicze\n",
      "missing word: foot-high\n",
      "missing word: nemy\n",
      "missing word: casually-dressed\n",
      "missing word: shley\n",
      "missing word: shushes\n",
      "missing word: souped-up\n",
      "missing word: put-out\n",
      "missing word: tousles\n",
      "missing word: portacabin\n",
      "missing word: c.m\n",
      "missing word: vening\n",
      "missing word: gun-like\n",
      "missing word: turban-like\n",
      "missing word: snake-topped\n",
      "missing word: overbalances\n",
      "missing word: well-wrapped\n",
      "missing word: fat-faced\n",
      "missing word: four-poster\n",
      "missing word: aragog\n",
      "missing word: swips\n",
      "missing word: threshes\n",
      "missing word: feelss\n",
      "missing word: grides\n",
      "missing word: thestral\n",
      "missing word: thestrals\n",
      "missing word: earthwards\n",
      "missing word: deluminator\n",
      "missing word: disapparates\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "missing word: grimmauld\n",
      "missing word: polyjuice\n",
      "missing word: nervous-looking\n",
      "missing word: up-key\n",
      "missing word: long-boned\n",
      "missing word: imagistically\n",
      "missing word: roof-less\n",
      "missing word: disapparate\n",
      "missing word: frost-covered\n",
      "missing word: crypt-like\n",
      "missing word: wand-maker\n",
      "missing word: delimunator\n",
      "missing word: apparates\n",
      "missing word: un-carpeted\n",
      "missing word: proclaming\n",
      "missing word: durmstrang\n",
      "missing word: owlery\n",
      "missing word: horntail\n",
      "missing word: hogwards\n",
      "missing word: dreagon\n",
      "missing word: griffindor\n",
      "missing word: beauxbatons\n",
      "missing word: fur-trimmed\n",
      "missing word: gillyweed\n",
      "missing word: rocks-strewn\n",
      "missing word: merperson\n",
      "missing word: merpeople\n",
      "missing word: grindylows\n",
      "missing word: heaped-in\n",
      "missing word: tadpole-like\n",
      "missing word: pensieve\n",
      "missing word: riving\n",
      "missing word: black-robed\n",
      "missing word: dementor\n",
      "missing word: -eaten\n",
      "missing word: crookshanks\n",
      "missing word: sugar-pink\n",
      "missing word: half-horse\n",
      "missing word: thwacks\n",
      "missing word: o.w.l\n",
      "missing word: shoulder-to-shoulder\n",
      "missing word: under-station\n",
      "missing word: stern-looking\n",
      "missing word: ollivanders\n",
      "missing word: disserted\n",
      "missing word: pitch-\n",
      "missing word: first-years\n",
      "missing word: gryffindors\n",
      "missing word: half-severed\n",
      "missing word: guilt-framed\n",
      "missing word: -silk\n",
      "missing word: wide-stone\n",
      "missing word: remembrall\n",
      "missing word: hackling\n",
      "missing word: torch-lit\n",
      "missing word: unchains\n",
      "missing word: bludgers\n",
      "missing word: crouniers\n",
      "missing word: red-\n",
      "missing word: erised\n",
      "missing word: triple-decker\n",
      "missing word: scabbers\n",
      "missing word: relights\n",
      "missing word: buckbeak\n",
      "missing word: half-eagle\n",
      "missing word: roller-skates\n",
      "missing word: wide-awake\n",
      "missing word: honeydukes\n",
      "missing word: half-wakes\n",
      "missing word: hogwart\n",
      "missing word: outmaneuvers\n",
      "missing word: rain-lashed\n",
      "missing word: unsmilingly\n",
      "missing word: hefts\n",
      "missing word: white-coated\n",
      "missing word: pasted-in\n",
      "missing word: leaf-strewn\n",
      "missing word: half-run\n",
      "missing word: half-stumble\n",
      "missing word: snake-wranglers\n",
      "missing word: bow-string\n",
      "missing word: snake-charmer\n",
      "missing word: -smoking\n",
      "missing word: cone-\n",
      "missing word: rifled-butted\n",
      "missing word: bandoleers\n",
      "missing word: tailboard\n",
      "missing word: vicious-looking\n",
      "missing word: eye-line\n",
      "missing word: still-moving\n",
      "missing word: spirit-type\n",
      "missing word: hastily-\n",
      "missing word: cross-hairs\n",
      "missing word: head-\n",
      "missing word: rifle-butts\n",
      "missing word: -gun\n",
      "missing word: machine-gunner\n",
      "missing word: six-bar\n",
      "missing word: addresseses\n",
      "missing word: institutional-\n",
      "missing word: shaven-headed\n",
      "missing word: seedy-\n",
      "missing word: good-humoredly\n",
      "missing word: specter-like\n",
      "missing word: drowsily\n",
      "missing word: blood-smeared\n",
      "missing word: ever-persistent\n",
      "missing word: halterneck\n",
      "missing word: neon-lit\n",
      "missing word: blissful-looking\n",
      "missing word: flat-lining\n",
      "missing word: now-untarnished\n",
      "missing word: chest-high\n",
      "missing word: introspectively\n",
      "missing word: half-four\n",
      "missing word: brick-walled\n",
      "missing word: consolingly\n",
      "missing word: rain-sodden\n",
      "missing word: head-butt\n",
      "missing word: test-card\n",
      "missing word: lead-footed\n",
      "missing word: gloatingly\n",
      "missing word: model-type\n",
      "missing word: expensive-looking\n",
      "missing word: comfortable-looking\n",
      "missing word: elias-clark\n",
      "missing word: marble-floored\n",
      "missing word: webcamming\n",
      "missing word: grey-\n",
      "missing word: coiffured\n",
      "missing word: ungraciously\n",
      "missing word: slim-fitting\n",
      "missing word: notredam\n",
      "missing word: o-style\n",
      "missing word: splutters\n",
      "missing word: high-fives\n",
      "missing word: serious-looking\n",
      "missing word: free-falls\n",
      "missing word: sparta-themed\n",
      "missing word: burnt-rubber\n",
      "missing word: roller-blade\n",
      "missing word: rail-cart\n",
      "missing word: chimneypots\n",
      "missing word: two-handled\n",
      "missing word: stone-walled\n",
      "missing word: gold-framed\n",
      "missing word: carrows\n",
      "missing word: demetor\n",
      "missing word: nut-nutted\n",
      "missing word: dis-apparate\n",
      "missing word: pale-faced\n",
      "missing word: unclenches\n",
      "missing word: gaunt-faced\n",
      "missing word: unbruised\n",
      "missing word: oldemort\n",
      "missing word: dis-apparates\n",
      "missing word: craddles\n",
      "missing word: rubble-\n",
      "missing word: whiz-bang\n",
      "missing word: coin-sized\n",
      "missing word: stern-faced\n",
      "missing word: outliners\n",
      "missing word: night-\n",
      "missing word: hurddle\n",
      "missing word: fasade\n",
      "missing word: silhouetting\n",
      "missing word: metallic-skinned\n",
      "missing word: sarcophagus-like\n",
      "missing word: lumbs\n",
      "missing word: curles\n",
      "missing word: heavily-armed\n",
      "missing word: remote-piloted\n",
      "missing word: rain-drenched\n",
      "missing word: concrete-walled\n",
      "missing word: half-destroyed\n",
      "missing word: car-bed\n",
      "missing word: officiously\n",
      "missing word: slo-mo\n",
      "missing word: semi-crowded\n",
      "missing word: ill-behaved\n",
      "missing word: lornings\n",
      "missing word: unlandscaped\n",
      "missing word: worse-case\n",
      "missing word: slushie\n",
      "missing word: tic-tacs\n",
      "missing word: half-nelson\n",
      "missing word: ungracefully\n",
      "missing word: knocked-up\n",
      "missing word: unshowered\n",
      "missing word: dog.it\n",
      "missing word: table.takes\n",
      "missing word: ever-so\n",
      "missing word: roll-away\n",
      "missing word: lawn.two\n",
      "missing word: changesthe\n",
      "missing word: 'security\n",
      "missing word: beer-guzzling\n",
      "missing word: santa-in-the-off-season\n",
      "missing word: pillipina\n",
      "missing word: pruney\n",
      "missing word: key-box\n",
      "missing word: mirage-\n",
      "missing word: snot-nosed\n",
      "missing word: loosened-tie\n",
      "missing word: middle-management\n",
      "missing word: new-money\n",
      "missing word: palm-prints\n",
      "missing word: filterless\n",
      "missing word: santa-elf\n",
      "missing word: screeeeech\n",
      "missing word: nose-to-nose\n",
      "missing word: squeeeeeeeak\n",
      "missing word: creeeeeeeeak\n",
      "missing word: fishtails\n",
      "missing word: underneath.and\n",
      "missing word: pop-pop\n",
      "missing word: re-aims\n",
      "missing word: soldier-like\n",
      "missing word: dyed-black\n",
      "missing word: mini-skirt\n",
      "missing word: uhaul\n",
      "missing word: mp5s\n",
      "missing word: darkness.coming\n",
      "missing word: truck.a\n",
      "missing word: skeletally\n",
      "missing word: unhinges\n",
      "missing word: re-arms\n",
      "missing word: darkness.and\n",
      "missing word: opned\n",
      "missing word: baby.catching\n",
      "missing word: clockwork-like\n",
      "missing word: and.click\n",
      "missing word: baptistine\n",
      "missing word: someone-sur-mer\n",
      "missing word: clammering\n",
      "missing word: clean-\n",
      "missing word: organ-\n",
      "missing word: bowsprits\n",
      "missing word: down-and-\n",
      "missing word: thieved\n",
      "missing word: begs/taunts\n",
      "missing word: gorbeau\n",
      "missing word: clacquesous\n",
      "missing word: wrought-ironwork\n",
      "missing word: musain\n",
      "missing word: underlights\n",
      "missing word: richochet\n",
      "missing word: re-loaded\n",
      "missing word: too-big\n",
      "missing word: in-pain\n",
      "missing word: hand-cuffed\n",
      "missing word: 9-\n",
      "missing word: 14-\n",
      "missing word: twizzler\n",
      "missing word: screech-screech-\n",
      "missing word: awful-loud\n",
      "missing word: white-metal\n",
      "missing word: seat-belted\n",
      "missing word: screech-backs\n",
      "missing word: rrrrrrrroooooooooaaaaaarrrrrrrr\n",
      "missing word: lower-lower-middle\n",
      "missing word: heavying\n",
      "missing word: flipflipflips\n",
      "missing word: shudder-hum\n",
      "missing word: cracked-glass\n",
      "missing word: mid-ground\n",
      "missing word: slam-flattened\n",
      "missing word: cli-chunk\n",
      "missing word: unsafeties\n",
      "missing word: kafa-\n",
      "missing word: bus-sized\n",
      "missing word: shatter-crash\n",
      "missing word: baaam\n",
      "missing word: window-\n",
      "missing word: top-side\n",
      "missing word: semi-demolished\n",
      "missing word: dirt-caked\n",
      "missing word: nearly-dug\n",
      "missing word: rumbling/mechanical\n",
      "missing word: rock-wall\n",
      "missing word: ffft\n",
      "missing word: entertainment-leaning\n",
      "missing word: kphx\n",
      "missing word: e-harmony\n",
      "missing word: now-vibrating\n",
      "missing word: mid-argument\n",
      "missing word: ichats\n",
      "missing word: laviani\n",
      "missing word: surpressing\n",
      "missing word: fifty-five-year-old\n",
      "missing word: panicing\n",
      "missing word: paniced\n",
      "missing word: tri-wizard\n",
      "missing word: gilliweed\n",
      "missing word: 'people\n",
      "missing word: snackbowl\n",
      "missing word: veritaserum\n",
      "missing word: sunken-faced\n",
      "missing word: bookcovers\n",
      "missing word: flumph\n",
      "missing word: ragged-looking\n",
      "missing word: ommited\n",
      "missing word: amber-red\n",
      "missing word: ever-so-slightly\n",
      "missing word: swook\n",
      "missing word: pale-eyed\n",
      "missing word: jet-someone\n",
      "missing word: dust-laden\n",
      "missing word: mirthlessly\n",
      "missing word: unloops\n",
      "missing word: owwwwwwww\n",
      "missing word: poising\n",
      "count of missing words:  2312\n",
      "18624 18626 18626\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')\n",
    "\n",
    "vocab, total_len = {}, 0\n",
    "for cap in train_corpus:\n",
    "    tokens = nltk.word_tokenize(cap.lower())\n",
    "    total_len += len(tokens)\n",
    "    for w in tokens:\n",
    "        try:\n",
    "            vocab[w] += 1\n",
    "        except:\n",
    "            vocab[w] = 1\n",
    "\n",
    "print('Avg. count of words per caption:', total_len/len(train_corpus))\n",
    "print('Count of unique words: ', len(vocab))\n",
    "\n",
    "to_del = []\n",
    "for w in vocab.keys():\n",
    "    if not w in wordvectors:\n",
    "        to_del.append(w)\n",
    "        print('missing word: {}'.format(w))\n",
    "\n",
    "print('count of missing words: ', len(to_del))\n",
    "        \n",
    "for w in to_del:\n",
    "    del vocab[w]\n",
    "        \n",
    "idx2word = {idx: word for idx, word in enumerate(['<eos>', '<unk>'] + list(vocab.keys()))}\n",
    "word2idx = {word: idx for idx, word in enumerate(['<eos>', '<unk>'] + list(vocab.keys()))}\n",
    "EOS, UNK = 0, 1\n",
    "\n",
    "print(len(vocab), len(idx2word), len(word2idx))\n",
    "\n",
    "word_embeddings = np.zeros((len(idx2word), 300))\n",
    "for idx, word in idx2word.items():\n",
    "    if idx == EOS:\n",
    "        word_embeddings[idx] = wordvectors['eos']\n",
    "    elif idx == UNK:\n",
    "        word_embeddings[idx] = wordvectors['unk']\n",
    "    else:\n",
    "        word_embeddings[idx] = wordvectors[word]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine POS-tagging vocabulary from train split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique words per tag:\n",
      " PRP$:\t7\n",
      " NN:\t9390\n",
      " NNS:\t3628\n",
      " IN:\t160\n",
      " DT:\t26\n",
      " .:\t3\n",
      " VBZ:\t1846\n",
      " PRP:\t27\n",
      " CC:\t25\n",
      " RP:\t29\n",
      " VBP:\t1190\n",
      " VBG:\t1703\n",
      " ,:\t1\n",
      " RB:\t1344\n",
      " JJ:\t5475\n",
      " ::\t4\n",
      " TO:\t2\n",
      " VB:\t1389\n",
      " POS:\t7\n",
      " VBN:\t1387\n",
      " VBD:\t1093\n",
      " JJR:\t82\n",
      " (:\t1\n",
      " ):\t1\n",
      " PDT:\t11\n",
      " WP:\t8\n",
      " MD:\t19\n",
      " WRB:\t8\n",
      " WDT:\t7\n",
      " RBR:\t36\n",
      " CD:\t164\n",
      " JJS:\t45\n",
      " RBS:\t3\n",
      " EX:\t1\n",
      " FW:\t21\n",
      " WP$:\t1\n",
      " NNP:\t19\n",
      " '':\t2\n",
      " UH:\t4\n",
      " SYM:\t6\n",
      " ``:\t1\n",
      " $:\t1\n",
      "44\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "pos_vocab = {}\n",
    "pos_unique_words = {}\n",
    "for cap in train_corpus:\n",
    "    for tag in nltk.pos_tag(nltk.word_tokenize(cap.lower())):\n",
    "        try:\n",
    "            pos_vocab[tag[1]] += 1\n",
    "            try: \n",
    "                pos_unique_words[tag[1]][tag[0]] += 1\n",
    "            except:\n",
    "                pos_unique_words[tag[1]][tag[0]] = 1\n",
    "        except:\n",
    "            pos_vocab[tag[1]] = 1\n",
    "            pos_unique_words[tag[1]] = {tag[0]: 1}\n",
    "\n",
    "print('Unique words per tag:')\n",
    "print('\\n'.join([f' {k}:\\t{len(words)}' for k, words in pos_unique_words.items()]))\n",
    "            \n",
    "idx2pos = {idx: tag for idx, tag in enumerate(['eos', 'unk'] + list(pos_vocab.keys()))}\n",
    "pos2idx = {tag: idx for idx, tag in enumerate(['eos', 'unk'] + list(pos_vocab.keys()))}\n",
    "EOS, UNK = 0, 1\n",
    "print(len(idx2pos))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine Universal POS-tagging from train split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package universal_tagset to\n",
      "[nltk_data]     /home/jeperez/nltk_data...\n",
      "[nltk_data]   Package universal_tagset is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique words per universal tag:\n",
      " PRON:\t42\n",
      " NOUN:\t12781\n",
      " ADP:\t160\n",
      " DET:\t41\n",
      " .:\t14\n",
      " VERB:\t6842\n",
      " CONJ:\t25\n",
      " PRT:\t38\n",
      " ADV:\t1377\n",
      " ADJ:\t5586\n",
      " NUM:\t164\n",
      " X:\t31\n",
      "14\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('universal_tagset')\n",
    "\n",
    "upos_vocab = {}\n",
    "upos_unique_words = {}\n",
    "for cap in train_corpus:\n",
    "    for tag in nltk.pos_tag(nltk.word_tokenize(cap.lower()), tagset='universal'):\n",
    "        try:\n",
    "            upos_vocab[tag[1]] += 1\n",
    "            try: \n",
    "                upos_unique_words[tag[1]][tag[0]] += 1\n",
    "            except:\n",
    "                upos_unique_words[tag[1]][tag[0]] = 1\n",
    "        except:\n",
    "            upos_vocab[tag[1]] = 1\n",
    "            upos_unique_words[tag[1]] = {tag[0]: 1}\n",
    "\n",
    "print('Unique words per universal tag:')\n",
    "print('\\n'.join([f' {k}:\\t{len(words)}' for k, words in upos_unique_words.items()]))\n",
    "            \n",
    "idx2upos = {idx: word for idx, word in enumerate(['eos', 'unk'] + list(upos_vocab.keys()))}\n",
    "upos2idx = {word: idx for idx, word in enumerate(['eos', 'unk'] + list(upos_vocab.keys()))}\n",
    "print(len(idx2upos))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate ground-truth references files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../results/20B-SS-v2_val_references.txt', 'w') as f:\n",
    "    for vidx, cap in zip(valid_vidxs, valid_corpus):\n",
    "        f.write('{}\\t{}\\n'.format(vidx, cap.lower()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate corpus.pkl file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "train_corpus_widxs = [[word2idx[w] if w in vocab else UNK for w in nltk.word_tokenize(cap.lower())] + [EOS] for cap in train_corpus]\n",
    "valid_corpus_widxs = [[word2idx[w] if w in vocab else UNK for w in nltk.word_tokenize(cap.lower())] + [EOS] for cap in valid_corpus]\n",
    "\n",
    "train_corpus_pidxs = [[pos2idx[w[1]] if w[1] in pos_vocab else UNK for w in nltk.pos_tag(nltk.word_tokenize(cap.lower()))] + [EOS] for cap in train_corpus]\n",
    "valid_corpus_pidxs = [[pos2idx[w[1]] if w[1] in pos_vocab else UNK for w in nltk.pos_tag(nltk.word_tokenize(cap.lower()))] + [EOS] for cap in valid_corpus]\n",
    "\n",
    "assert len(train_corpus_widxs) == len(train_vidxs) and len(train_vidxs) == len(train_corpus_pidxs) and len(train_vidxs) == len(train_corpus)\n",
    "assert len(valid_corpus_widxs) == len(valid_vidxs) and len(valid_vidxs) == len(valid_corpus_pidxs) and len(valid_vidxs) == len(valid_corpus)\n",
    "\n",
    "train_data = [train_corpus_widxs, train_vidxs, train_corpus_pidxs, train_corpus]\n",
    "valid_data = [valid_corpus_widxs, valid_vidxs, valid_corpus_pidxs, valid_corpus]\n",
    "test_data = [None, test_vidxs, None]\n",
    "\n",
    "with open('../../../data/Something-Something-v2/20b-ss-v2_corpus_pos.pkl', 'wb') as outfile:\n",
    "    pickle.dump([train_data, valid_data, test_data, vocab, idx2word, word_embeddings, idx2pos], outfile)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}