2.0 - Training - Generating Expansion Candidates.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IMPORTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%load_ext cython"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import cfg\n",
    "\n",
    "import cPickle\n",
    "import gzip\n",
    "import json\n",
    "import os\n",
    "import pandas as pd\n",
    "import time\n",
    "from collections import OrderedDict\n",
    "\n",
    "import progress_bar as pb\n",
    "from efficient_query_expansion.normalize_text import normalize_text, get_stopword_set\n",
    "from efficient_query_expansion.query_expansion_support import QueryExpansionSupport"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LOAD ALL THE DOCUMENTS, QUERIES AND ASSOCIATIONS IN MEMORY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "def jsonConvertKeys(constructor):\n",
    "    return lambda x: dict((constructor(k),v) for k,v in x.iteritems())\n",
    "\n",
    "qid_to_query = json.load(open(cfg.training_dir + \"qid_to_query.json\", \"r\"), object_hook=jsonConvertKeys(int))\n",
    "qid_to_docid_list = json.load(open(cfg.training_dir + \"qid_to_docid_list.json\", \"r\"), object_hook=jsonConvertKeys(int))\n",
    "docid_to_rawtext = json.load(open(cfg.training_dir + \"docid_to_rawtext.json\", \"r\"), object_hook=jsonConvertKeys(long))\n",
    "\n",
    "assert len(qid_to_query) == len(qid_to_docid_list)\n",
    "assert all(docid in docid_to_rawtext for docid_list in qid_to_docid_list.itervalues() for docid in docid_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "docid_to_text = dict(\n",
    "    (docid, normalize_text(raw_text))\n",
    "    for docid, raw_text in pb.iteritems_progress(docid_to_rawtext)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# revert the qid_to_docid_list associations\n",
    "docid_to_qid_list = dict()\n",
    "for qid, docid_list in qid_to_docid_list.iteritems():\n",
    "    for docid in docid_list:\n",
    "        if docid in docid_to_qid_list:\n",
    "            docid_to_qid_list[docid].append(qid)\n",
    "        else:\n",
    "            docid_to_qid_list[docid] = [qid]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LOAD REWRITING STRATEGIES SUPPORT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%time expansion_support = cPickle.load(open(cfg.thesaurus_dir + \"expansion_support.pickle\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# this set of terms represents an overestimation of the good terms.\n",
    "# We filled it with all terms having document frequency greater than 20\n",
    "%time good_unary_terms = set(line.strip() for line in gzip.open(cfg.raw_dir + \"frequent_terms.txt.gz\"))\n",
    "print len(good_unary_terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# collection dependent term-statistics. This dictionary depends from the dataset\n",
    "\n",
    "# segment_to_phrase_freq contains the document frequency of each thesaurus' n-gram\n",
    "segment_to_phrase_freq = cPickle.load(open(cfg.thesaurus_dir + \"segment_to_phrase_freq.pickle\", \"r\"))\n",
    "\n",
    "# segment_to_and_freq contains the number of documents containing all terms of each thesaurus' n-gram.\n",
    "# to avoid duplicates the keys are collapsed according to: \" \".join(sorted(ngram.split()))\n",
    "segment_to_and_freq = cPickle.load(open(cfg.thesaurus_dir + \"segment_to_and_freq.pickle\", \"r\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "assert all(\n",
    "    segment in segment_to_phrase_freq and \" \".join(sorted(segment.split())) in segment_to_and_freq\n",
    "    for segment in expansion_support[\"segment_id_to_segment\"]\n",
    "    if \" \" in segment\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "stopwords = get_stopword_set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "qes = QueryExpansionSupport(expansion_support, good_unary_terms, stopwords, segment_to_phrase_freq, segment_to_and_freq)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# REWRITING STRATEGIES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# creates the query representation without any expansion\n",
    "def query_to_base(query):\n",
    "    global stopwords\n",
    "\n",
    "    # normalize and tokenize the text\n",
    "    query = normalize_text(query).split()\n",
    "\n",
    "    # remove the stop words, but if the query is composed only by stopwords use all original terms\n",
    "    query = filter((lambda x: x not in stopwords), query) or query\n",
    "\n",
    "    # simulate the \"synset\" to match the signature. The synset is composed only by the term istelf (and its tag)\n",
    "    base_query = map((lambda term: [(term,)]), query)\n",
    "    candidates = map((lambda term: []), query)\n",
    "\n",
    "    # the query is composed only of one \"segmentation\".\n",
    "    # In case of more segmentations the two arrays contain more CNF queries\n",
    "    return [base_query], [candidates]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "strategies = OrderedDict([\n",
    "    (\"Base\", query_to_base),\n",
    "    (\"SegmentedThesaurusExpansion\", qes.get_all_theraurus_expansions),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "strategy_name_to_qid_to_base_query = OrderedDict()\n",
    "strategy_name_to_qid_to_candidates = OrderedDict()\n",
    "keys = []\n",
    "table = []\n",
    "for strategy_name, strategy in strategies.iteritems():\n",
    "    strategy_name_to_qid_to_base_query[strategy_name] = qid_to_base_query = dict()\n",
    "    strategy_name_to_qid_to_candidates[strategy_name] = qid_to_candidates = dict()\n",
    "\n",
    "    start_time = time.time()\n",
    "    for qid, query in pb.iteritems_progress(qid_to_query, labeling_fun={\"prefix\":strategy_name}, hide_bar_on_success=True):\n",
    "        qid_to_base_query[qid], qid_to_candidates[qid] = strategy(query)\n",
    "    keys.append(strategy_name)\n",
    "    table.append([1.0 * (time.time()-start_time) / len(qid_to_base_query)])\n",
    "\n",
    "    del qid_to_candidates, qid_to_base_query\n",
    "# it lasts 7min 30s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(table, index=keys, columns=[\"Avg. expansions time\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# COMPUTE THE NUMBER OF MATCHES OF THE EXPANDED QUERIES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%cython\n",
    "\n",
    "def query_match(text, base_query, candidates):\n",
    "    return any(  # or level: at least one of the OR macro terms must match the text\n",
    "        all(  # and level: each synset must match the text\n",
    "            any(  # or level: at least one of the word in the synset must be in the text\n",
    "                (\" \" + word_and_tags[0] + \" \") in text\n",
    "                for word_and_tags in synset + candidates[i][j]\n",
    "            ) for j, synset in enumerate(and_query)\n",
    "        ) for i, and_query in enumerate(base_query)\n",
    "        if len(and_query) > 0\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute the number of matches of each rewrite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "strategy_name_to_qid_to_num_match = OrderedDict()\n",
    "\n",
    "for strategy_name in strategies:\n",
    "    strategy_name_to_qid_to_num_match[strategy_name] = dict((qid, 0) for qid in qid_to_docid_list)\n",
    "\n",
    "for docid, doc_text in pb.iteritems_progress(docid_to_text):\n",
    "    if docid not in docid_to_qid_list:\n",
    "        continue\n",
    "    for qid in docid_to_qid_list[docid]:\n",
    "        # for each strategy check if the query matchs the document\n",
    "        for strategy_name, qid_to_base_query in strategy_name_to_qid_to_base_query.iteritems():\n",
    "            strategy_name_to_qid_to_num_match[strategy_name][qid] += \\\n",
    "                query_match(doc_text, qid_to_base_query[qid], strategy_name_to_qid_to_candidates[strategy_name][qid])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# GROUND TRUTH BUILD (using the same format used previously)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "strategy_name = \"SegmentedThesaurusExpansion\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "assert strategy_name in strategies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# for the training of the models we consider only the queries having at least one candidate expansion that can improve its recall.\n",
    "queries_with_recall_improvement = [\n",
    "    qid\n",
    "    for qid, num_match in strategy_name_to_qid_to_num_match[strategy_name].iteritems()\n",
    "    if num_match > strategy_name_to_qid_to_num_match[\"Base\"][qid]\n",
    "]\n",
    "print len(queries_with_recall_improvement)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## COMPUTE THE WORD OCCURRENCES OF EACH QUERY, NEEDED BY THE TRAINING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%cython\n",
    "\n",
    "cdef _get_word_set(query_repr):\n",
    "    return set(\n",
    "        word_and_tags[0]\n",
    "        for and_query in query_repr\n",
    "        for synset in and_query\n",
    "        for word_and_tags in synset\n",
    "    )\n",
    "\n",
    "def compute_word_occurrence_set(base_query, candidates, docid_list, docid_to_text):\n",
    "    word_set = _get_word_set(base_query) | _get_word_set(candidates)\n",
    "\n",
    "    return dict(\n",
    "        (word, set(docid\n",
    "                   for docid in docid_list\n",
    "                   if (\" \" + word + \" \") in docid_to_text[docid]))\n",
    "        for word in word_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "qid_to_word_to_occurrence_set = dict()\n",
    "\n",
    "for qid in pb.iter_progress(qid_to_query):\n",
    "    if qid < 0:\n",
    "        continue\n",
    "    qid_to_word_to_occurrence_set[qid] = compute_word_occurrence_set(\n",
    "        strategy_name_to_qid_to_base_query[strategy_name][qid],\n",
    "        strategy_name_to_qid_to_candidates[strategy_name][qid],\n",
    "        qid_to_docid_list[qid],\n",
    "        docid_to_text\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SAVE THE GROUND TRUTH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if not os.path.isdir(cfg.processed_dir):\n",
    "    os.mkdir(cfg.processed_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "with open(cfg.processed_dir + \"queries_with_recall_improvement.pickle\", \"wb\") as outfile:\n",
    "    cPickle.dump(queries_with_recall_improvement, outfile, protocol=cPickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "with open(cfg.processed_dir + \"qid_to_word_to_occurrence_set.pickle\", \"wb\") as outfile:\n",
    "    cPickle.dump(qid_to_word_to_occurrence_set, outfile, protocol=cPickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "with open(cfg.processed_dir + \"qid_to_base_query.pickle\", \"wb\") as outfile:\n",
    "    cPickle.dump(\n",
    "        dict(\n",
    "            (qid, base_query)\n",
    "            for qid, base_query in strategy_name_to_qid_to_base_query[strategy_name].iteritems()\n",
    "            if qid in qid_to_query\n",
    "        ),\n",
    "        outfile,\n",
    "        protocol=cPickle.HIGHEST_PROTOCOL\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "with open(cfg.processed_dir + \"qid_to_candidates.pickle\", \"wb\") as outfile:\n",
    "    cPickle.dump(\n",
    "        dict(\n",
    "            (qid, candidates)\n",
    "            for qid, candidates in strategy_name_to_qid_to_candidates[strategy_name].iteritems()\n",
    "            if qid in qid_to_query\n",
    "        ),\n",
    "        outfile,\n",
    "        protocol=cPickle.HIGHEST_PROTOCOL\n",
    "    )"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "widgets": {
   "state": {
    "8f4e9f1abacd4d9592e211e9756dce5f": {
     "views": [
      {
       "cell_index": 40
      }
     ]
    },
    "ca17dc7e952a421fa5c9134752edaedf": {
     "views": [
      {
       "cell_index": 4
      }
     ]
    }
   },
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}