TMRetrieval_weighted.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Translation Memory Retrieval using Weighted N-Grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import math\n",
    "from collections import Counter\n",
    "import string\n",
    "import numpy as np\n",
    "import json\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/ashes/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "stop_words = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I request you to please remove the drive safely.\n",
      "i request you to please remove the drive safely.\n"
     ]
    }
   ],
   "source": [
    "input_line = input()\n",
    "\n",
    "sentence = input_line.lower()\n",
    "# sentence = \"I request you to please remove the drive safely.\"\n",
    "\n",
    "print(sentence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Weighted N-Gram Precision"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get sentences and IDF values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./tm_data/tm_src_2000_lower.txt\") as source_file:\n",
    "    sentences = source_file.read().splitlines()\n",
    "\n",
    "\n",
    "with open('./tm_data/idf_values_2000.json') as json_file:\n",
    "    idf_values_str = json.load(json_file)\n",
    "\n",
    "idf_values = ast.literal_eval(idf_values_str)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting the M_ngrams and C_ngrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_M_ngrams(sentence):\n",
    "    ngrams_list_sent = []\n",
    "    M_ngrams = []\n",
    "    counter_ngrams = []\n",
    "    \n",
    "    ngrams = list(nltk.ngrams(sentence.split(), 2))\n",
    "    ngrams_list_sent.append(list(ngrams))\n",
    "    M_ngrams = [y for x in ngrams_list_sent for y in x]\n",
    "    \n",
    "    for ngrams in M_ngrams:\n",
    "        counter_ngrams.append(Counter(ngrams))\n",
    "        \n",
    "    return M_ngrams\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "M_ngrams = get_M_ngrams(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_C_ngrams(candidate_sentence):\n",
    "    ngrams_list_sent = []\n",
    "    C_ngrams = []\n",
    "    counter_ngrams = []\n",
    "    \n",
    "    ngrams = list(nltk.ngrams(candidate_sentence.split(), 2))\n",
    "    ngrams_list_sent.append(list(ngrams))\n",
    "    C_ngrams = [y for x in ngrams_list_sent for y in x]\n",
    "    ngrams_sents = []\n",
    "    ngrams_list_sent = []\n",
    "    \n",
    "    for ngrams in C_ngrams:\n",
    "        counter_ngrams.append(Counter(ngrams))\n",
    "    \n",
    "    return C_ngrams\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To compute numerator and denominator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ngrams_intersection(candidate_sentence):\n",
    "    C_ngrams = get_C_ngrams(candidate_sentence)\n",
    "    \n",
    "    M_set = set(M_ngrams)\n",
    "    C_set = set(C_ngrams)\n",
    "    \n",
    "    return list(M_set & C_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_w_sum(ngrams_list):\n",
    "    w = 0\n",
    "    \n",
    "    for ngram in ngrams_list:\n",
    "        for token in ngram:\n",
    "            if token in idf_values:\n",
    "                w+= idf_values[token] \n",
    "    return w"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final score for each sentence wrt to input sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_wpn(candidate_sentence):\n",
    "    C_ngrams = get_C_ngrams(candidate_sentence)\n",
    "    intersection_ngrams = ngrams_intersection(candidate_sentence)\n",
    "    Z = 0.75\n",
    "    \n",
    "    w_M_ngrams = compute_w_sum(M_ngrams)\n",
    "    w_C_ngrams = compute_w_sum(C_ngrams)\n",
    "    w_intersection_ngrams = compute_w_sum(intersection_ngrams)\n",
    "    \n",
    "    \n",
    "    wpn = w_intersection_ngrams / ((Z*w_M_ngrams) + ((1-Z)*w_C_ngrams))\n",
    "    \n",
    "    return wpn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[811] we have sent down the book to you to make everything clear 0.06851292170457951\n",
      "[1133] here is cool water for you to wash in and drink 0.0715390087369681\n",
      "[1425] enabled tv screen that allows you to play pre 0.07441939850239553\n",
      "[103] and had he showed them to you to be numerous 0.074573352760082\n",
      "[208] safely remove the selected drive 0.12471474617158834\n"
     ]
    }
   ],
   "source": [
    "max_wpn = 0\n",
    "wnp_all = []\n",
    "N = 5\n",
    "\n",
    "for sentence in sentences:\n",
    "    wpn = compute_wpn(sentence)\n",
    "    wnp_all.append(wpn)\n",
    "    if wpn > max_wpn:\n",
    "        max_wpn = wpn\n",
    "        best_sentence = sentence\n",
    "            \n",
    "        \n",
    "wnp_all = np.array(wnp_all)\n",
    "sorted_indices = np.argsort(wnp_all) \n",
    "least_N_indices = sorted_indices[-N:] \n",
    "\n",
    "print()\n",
    "for i in least_N_indices:\n",
    "    print([i+1], sentences[i], wnp_all[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieval of Target from TM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[811] हमने तुमपर किताब अवतरित की हर चीज़ को खोलकर बयान करने के लिए और मुस्लिम (आज्ञाकारियों) के लिए मार्गदर्शन\n",
      "\n",
      "[1133] \n",
      "\n",
      "[1425] इन दरवाजों का नियंत्रण ट्रेन के गार्ड के पास रखा गया है जिससे यात्रियों की सुरक्षा और सुदृढ़ हुई है।\n",
      "\n",
      "[103] निश्चय ही वह तो जो कुछ दिलों में होता है उसे भी जानता है\n",
      "\n",
      "[208] चयनित ड्राइव सुरक्षित रूप से निकालें\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tgt_tm_array = []\n",
    "\n",
    "with open('./tm_data/tm_tgt_2000.txt') as tgt_tm:\n",
    "    line = tgt_tm.readline()\n",
    "    \n",
    "    while line:\n",
    "        tgt_tm_array.append(line)\n",
    "        line = tgt_tm.readline()\n",
    "  \n",
    "    for i in least_N_indices:\n",
    "        print([i+1], tgt_tm_array[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}