rule.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "import editdistance\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pytorch_pretrained_bert.tokenization import BertTokenizer\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from spacy.tokenizer import Tokenizer\n",
    "from spacy.lang.en import English\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "import revtok\n",
    "import string\n",
    "import tempfile\n",
    "from tqdm import tqdm\n",
    "\n",
    "from evaluator import evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file = 'sharc1-official/json/sharc_dev.json'\n",
    "# test_file = '../adverse-datasets/sharc_dev_shuffled.json'\n",
    "# test_file = '../adverse-datasets/sharc_dev_regular.json'\n",
    "# test_file = '../adverse-datasets/sharc_dev_augmented.json'\n",
    "\n",
    "\n",
    "with open(test_file) as file:\n",
    "    dev_dataset = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 69/69 [00:10<00:00,  6.67it/s]\n"
     ]
    }
   ],
   "source": [
    "BERT_VOCAB = 'bert-base-uncased-vocab.txt'\n",
    "LOWERCASE = True\n",
    "tokenizer = BertTokenizer.from_pretrained(BERT_VOCAB, do_lower_case=LOWERCASE, cache_dir=None)\n",
    "MATCH_IGNORE = {'do', 'have', '?'}\n",
    "SPAN_IGNORE = set(string.punctuation)\n",
    "\n",
    "def tokenize(doc):\n",
    "    if not doc.strip():\n",
    "        return []\n",
    "    tokens = []\n",
    "    for i, t in enumerate(revtok.tokenize(doc)):\n",
    "        subtokens = tokenizer.tokenize(t.strip())\n",
    "        for st in subtokens:\n",
    "            tokens.append({\n",
    "                'orig': t,\n",
    "                'sub': st,\n",
    "                'orig_id': i,\n",
    "            })\n",
    "    return tokens\n",
    "\n",
    "def filter_answer(answer):\n",
    "    return detokenize([a for a in answer if a['orig'] not in MATCH_IGNORE])\n",
    "\n",
    "def filter_chunk(answer):\n",
    "    return detokenize([a for a in answer if a['orig'] not in MATCH_IGNORE])\n",
    "\n",
    "def detokenize(tokens):\n",
    "    words = []\n",
    "    for i, t in enumerate(tokens):\n",
    "        if t['orig_id'] is None or (i and t['orig_id'] == tokens[i-1]['orig_id']):\n",
    "            continue\n",
    "        else:\n",
    "            words.append(t['orig'])\n",
    "    return revtok.detokenize(words)\n",
    "\n",
    "def get_span(context, answer):\n",
    "    answer = filter_answer(answer)\n",
    "    best, best_score = None, float('inf')\n",
    "    stop = False\n",
    "    for i in range(len(context)):\n",
    "        if stop:\n",
    "            break\n",
    "        for j in range(i, len(context)):\n",
    "            chunk = filter_chunk(context[i:j+1])\n",
    "            if '\\n' in chunk or '*' in chunk:\n",
    "                continue\n",
    "            score = editdistance.eval(answer, chunk)\n",
    "            if score < best_score or (score == best_score and j-i < best[1]-best[0]):\n",
    "                best, best_score = (i, j), score\n",
    "            if chunk == answer:\n",
    "                stop = True\n",
    "                break\n",
    "    s, e = best\n",
    "    while not context[s]['orig'].strip() or context[s]['orig'] in SPAN_IGNORE:\n",
    "        s += 1\n",
    "    while not context[e]['orig'].strip() or context[s]['orig'] in SPAN_IGNORE:\n",
    "        e -= 1\n",
    "    return s, e\n",
    "\n",
    "def get_bullets(context):\n",
    "    indices = [i for i, c in enumerate(context) if c == '*']\n",
    "    pairs = list(zip(indices, indices[1:] + [len(context)]))\n",
    "    cleaned = []\n",
    "    for s, e in pairs:\n",
    "        while not context[e-1].strip():\n",
    "            e -= 1\n",
    "        while not context[s].strip() or context[s] == '*':\n",
    "            s += 1\n",
    "        if e - s > 2 and e - 2 < 45:\n",
    "            cleaned.append((s, e-1))\n",
    "    return cleaned\n",
    "\n",
    "def extract_clauses(data, tokenizer):\n",
    "    snippet = data['snippet']\n",
    "    t_snippet = tokenize(snippet)\n",
    "    questions = data['questions']\n",
    "    t_questions = [tokenize(q) for q in questions]\n",
    "\n",
    "    spans = [get_span(t_snippet, q) for q in t_questions]\n",
    "    bullets = get_bullets(t_snippet)\n",
    "    all_spans = spans + bullets\n",
    "    coverage = [False] * len(t_snippet)\n",
    "    sorted_by_len = sorted(all_spans,  key=lambda tup: tup[1] - tup[0], reverse=True)\n",
    "\n",
    "    ok = []\n",
    "    for s, e in sorted_by_len:\n",
    "        if not all(coverage[s:e+1]):\n",
    "            for i in range(s, e+1):\n",
    "                coverage[i] = True\n",
    "            ok.append((s, e))\n",
    "    ok.sort(key=lambda tup: tup[0])\n",
    "\n",
    "    match = {}\n",
    "    match_text = {}\n",
    "    clauses = [None] * len(ok)\n",
    "    for q, tq in zip(questions, t_questions):\n",
    "        best_score = float('inf')\n",
    "        best = None\n",
    "        for i, (s, e) in enumerate(ok):\n",
    "            score = editdistance.eval(detokenize(tq), detokenize(t_snippet[s:e+1]))\n",
    "            if score < best_score:\n",
    "                best_score, best = score, i\n",
    "                clauses[i] = tq\n",
    "        match[q] = best\n",
    "        s, e = ok[best]\n",
    "        match_text[q] = detokenize(t_snippet[s:e+1])\n",
    "\n",
    "    return {'questions': {q: tq for q, tq in zip(questions, t_questions)}, 'snippet': snippet, 't_snippet': t_snippet, 'spans': ok, 'match': match, 'match_text': match_text, 'clauses': clauses}\n",
    "\n",
    "with open(test_file) as f:\n",
    "    data = json.load(f)\n",
    "    tasks = {}\n",
    "    for ex in data:\n",
    "        for h in ex['evidence']:\n",
    "            if 'followup_question' in h:\n",
    "                h['follow_up_question'] = h['followup_question']\n",
    "                h['follow_up_answer'] = h['followup_answer']\n",
    "                del h['followup_question']\n",
    "                del h['followup_answer']\n",
    "        if ex['tree_id'] in tasks:\n",
    "            task = tasks[ex['tree_id']]\n",
    "        else:\n",
    "            task = tasks[ex['tree_id']] = {'snippet': ex['snippet'], 'questions': set()}\n",
    "        for h in ex['history'] + ex['evidence']:\n",
    "            task['questions'].add(h['follow_up_question'])\n",
    "    keys = sorted(list(tasks.keys()))\n",
    "    vals = [extract_clauses(tasks[k], tokenizer) for k in tqdm(keys)]\n",
    "    trees_dev = {k: v for k, v in zip(keys, vals)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = English()\n",
    "nlp.add_pipe(nlp.create_pipe('sentencizer')) # updated\n",
    "tokenizer = nlp.Defaults.create_tokenizer(nlp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Miscelleaneous functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text):\n",
    "    return [token.text for token in tokenizer(text)]\n",
    "\n",
    "def prettify_utterance(utterance, predicted_answer=None):\n",
    "    output = 'RULE TEXT: ' + utterance['snippet'] + '\\n'\n",
    "    output += 'SCENARIO: ' + utterance['scenario'] + '\\n'          \n",
    "    output += 'QUESTION: ' + utterance['question'] + '\\n'\n",
    "    output += 'HISTORY: ' + history_to_string(utterance['history']) + '\\n'\n",
    "    output += 'GOLD ANSWER: ' + utterance['answer'] + '\\n'\n",
    "    if predicted_answer:\n",
    "        output += 'PREDICTED ANSWER: ' + str(predicted_answer)\n",
    "    return output\n",
    "\n",
    "def history_to_string(history):\n",
    "    output = ''\n",
    "    first_qa = True\n",
    "    for qa in history:\n",
    "        if not first_qa:\n",
    "            output += '\\n'\n",
    "        output += 'Q: ' + qa['follow_up_question'] + '\\n'\n",
    "        output += 'A: ' + qa['follow_up_answer']\n",
    "        first_qa = False\n",
    "    return output\n",
    "\n",
    "def get_action(answer):\n",
    "    return answer if answer in ['Yes', 'No', 'Irrelevant'] else 'More'\n",
    "\n",
    "def evaluate_model(model_fn, dataset, print_confusion_matrix_turn_wise=False):\n",
    "    prediction_json = []\n",
    "    gold_json = []\n",
    "    \n",
    "    for utterance in dataset:\n",
    "        prediction_json.append({'utterance_id': utterance['utterance_id'], 'answer': model_fn(utterance)})\n",
    "        gold_json.append({'utterance_id': utterance['utterance_id'], 'answer': utterance['answer']})\n",
    "        \n",
    "\n",
    "    if print_confusion_matrix_turn_wise:\n",
    "        for turn_number in range(1, 6):\n",
    "            predicted_actions = [get_action(x['answer']) for x in prediction_json \n",
    "                                 if len(id_map[x['utterance_id']]['history']) == turn_number - 1]\n",
    "            gold_actions = [get_action(x['answer']) for x in gold_json\n",
    "                            if len(id_map[x['utterance_id']]['history']) == turn_number - 1]\n",
    "\n",
    "            print(f\"Turn number: {turn_number}\")\n",
    "            print(confusion_matrix(gold_actions, predicted_actions, labels=['Irrelevant', 'More', 'No', 'Yes']))\n",
    "            print('\\n\\n')\n",
    "            \n",
    "    \n",
    "        \n",
    "    gold_file = tempfile.NamedTemporaryFile('w')\n",
    "    json.dump(gold_json, gold_file)\n",
    "    gold_file.seek(0)\n",
    "\n",
    "    prediction_file = tempfile.NamedTemporaryFile('w')\n",
    "    json.dump(prediction_json, prediction_file)\n",
    "    prediction_file.seek(0)\n",
    "    \n",
    "    return evaluate(gold_file.name, prediction_file.name, mode='combined')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Turn wise class distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Turn number: 1\n",
      "More          319\n",
      "Yes           156\n",
      "No            148\n",
      "Irrelevant    138\n",
      "dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "Turn number: 2\n",
      "Yes     302\n",
      "No      290\n",
      "More    165\n",
      "dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "Turn number: 3\n",
      "Yes     209\n",
      "No      200\n",
      "More     60\n",
      "dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "Turn number: 4\n",
      "Yes     104\n",
      "No       87\n",
      "More     18\n",
      "dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "Turn number: 5\n",
      "No     41\n",
      "Yes    33\n",
      "dtype: int64\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for turn_number in range(1, 6):\n",
    "    actions = [get_action(utterance['answer']) for utterance in dev_dataset if len(utterance['history']) == turn_number - 1]\n",
    "    print(f\"Turn number: {turn_number}\")\n",
    "    print(pd.Series(actions).value_counts())\n",
    "    print(\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Last follow up answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Turn number: 2\n",
      "True     422\n",
      "False    170\n",
      "dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "Turn number: 3\n",
      "True     317\n",
      "False     92\n",
      "dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "Turn number: 4\n",
      "True     170\n",
      "False     21\n",
      "dtype: int64\n",
      "\n",
      "\n",
      "\n",
      "Turn number: 5\n",
      "True    74\n",
      "dtype: int64\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for turn_number in range(2, 6):\n",
    "    truth = [utterance['answer'] == utterance['history'][-1]['follow_up_answer'] for utterance in dev_dataset \n",
    "             if utterance['answer'] in ['Yes', 'No'] and len(utterance['history']) == turn_number - 1]\n",
    "    print(f\"Turn number: {turn_number}\")\n",
    "    print(pd.Series(truth).value_counts())\n",
    "    print(\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scenario and History empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True     138\n",
      "False     69\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "truth = [utterance['answer'] == 'Irrelevant' for utterance in dev_dataset \n",
    "         if utterance['scenario'] == '' and utterance['history'] == []]\n",
    "print(pd.Series(truth).value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Distribution model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def distribution_model(utterance):\n",
    "    rule = utterance['snippet']\n",
    "    history = utterance['history']\n",
    "    scenario = utterance['scenario']\n",
    "    question = utterance['question']\n",
    "    \n",
    "    \n",
    "    turn_number = len(history) + 1\n",
    "    \n",
    "    if turn_number == 1:\n",
    "        if history == [] and scenario == '':\n",
    "            answer = 'Irrelevant'\n",
    "        else:\n",
    "            answer = rule\n",
    "    else:\n",
    "        answer = history[-1]['follow_up_answer']\n",
    "        \n",
    "    return answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'micro_accuracy': 0.604,\n",
       " 'macro_accuracy': 0.6744,\n",
       " 'bleu_1': 0.1406,\n",
       " 'bleu_2': 0.1171,\n",
       " 'bleu_3': 0.1017,\n",
       " 'bleu_4': 0.0892}"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_model(distribution_model, dev_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Smart model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relevant_query(text, query, threshold=0.5):\n",
    "    query_tokens = tokenize(query.lower())\n",
    "    text_tokens = set(tokenize(text.lower()))\n",
    "    \n",
    "    relevant_tokens = 0\n",
    "    total_tokens = 0\n",
    "    \n",
    "    for token in query_tokens:\n",
    "        if token in STOP_WORDS or token in string.punctuation:\n",
    "            continue\n",
    "        elif token in text_tokens:\n",
    "            relevant_tokens += 1\n",
    "        total_tokens += 1\n",
    "    \n",
    "    return (relevant_tokens / total_tokens) >= threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def number_rules(rule):\n",
    "    if '*' in rule: # bullet points\n",
    "        return rule.count('*')\n",
    "    else:\n",
    "        return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def next_follow_up(utterance):\n",
    "    previous_questions = set([x['follow_up_question'] for x in utterance['history']])\n",
    "    \n",
    "    tree = trees_dev[utterance['tree_id']]\n",
    "    dic = {}\n",
    "    for k, v in tree['match'].items():\n",
    "        if v in dic:\n",
    "            dic[v].add(k)\n",
    "        else:\n",
    "            dic[v] = {k}\n",
    "    match = {tuple(v): tree['match_text'][list(v)[0]] for k, v in sorted(dic.items())}\n",
    "    \n",
    "    for questions_set, clause in match.items():\n",
    "        if not any(question in previous_questions for question in questions_set):\n",
    "            return 'Are you ' + clause + '?'\n",
    "    return utterance['snippet']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model(utterance):\n",
    "    rule = utterance['snippet']\n",
    "    history = utterance['history']\n",
    "    scenario = utterance['scenario']\n",
    "    question = utterance['question']\n",
    "    turn_number = len(history) + 1\n",
    "    \n",
    "    if turn_number == 1:\n",
    "        if not scenario and not relevant_query(rule, question):\n",
    "            return 'Irrelevant'\n",
    "        else:\n",
    "            return next_follow_up(utterance)\n",
    "    elif turn_number == 2:\n",
    "        if (not scenario and number_rules(rule) >= turn_number) or (scenario and number_rules(rule) - 1 >= turn_number):\n",
    "            return next_follow_up(utterance)\n",
    "        else:\n",
    "            return history[-1]['follow_up_answer']\n",
    "    else:\n",
    "        return history[-1]['follow_up_answer']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'micro_accuracy': 0.6374,\n",
       " 'macro_accuracy': 0.7125,\n",
       " 'bleu_1': 0.6397,\n",
       " 'bleu_2': 0.5624,\n",
       " 'bleu_3': 0.5117,\n",
       " 'bleu_4': 0.4778}"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_model(model, dev_dataset, print_confusion_matrix_turn_wise=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}