TestCaseSuite_CallingOutBluff.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Reading test data of a particular prompt and finding average length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    0                                                  1\n",
      "0   8  Dear local newspaper, I think effects computer...\n",
      "1  10  Dear Local Newspaper, @CAPS1 I have found that...\n",
      "2   9  Dear reader, @ORGANIZATION1 has had a dramatic...\n",
      "3  10  I agree that computers deffinately are an adva...\n",
      "4   8  Dear local Newspaper, @CAPS1 in the society we...\n",
      "   rating                                               text\n",
      "0       8  Dear local newspaper, I think effects computer...\n",
      "1      10  Dear Local Newspaper, @CAPS1 I have found that...\n",
      "2       9  Dear reader, @ORGANIZATION1 has had a dramatic...\n",
      "3      10  I agree that computers deffinately are an adva...\n",
      "4       8  Dear local Newspaper, @CAPS1 in the society we...\n",
      "22.330532212885153\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "\n",
    "from nltk import tokenize\n",
    "prompt_name = 'Prompt1.csv'\n",
    "data_test = pd.read_csv(prompt_name,header =None)\n",
    "data_test = data_test[[0,1]]\n",
    "print(data_test.head())\n",
    "data_test.columns = ['rating',\"text\"]\n",
    "P= 1 #prompt number\n",
    "\n",
    "\n",
    "#Calculates the average length of responses on that particular prompt\n",
    "def avg_len(data_test):\n",
    "    \n",
    "    avg_len=[]\n",
    "    for i, r in data_test.iterrows():\n",
    "        text = nltk.sent_tokenize(r['text'])\n",
    "        avg_len.append(len(text))\n",
    "    return (np.mean(avg_len))\n",
    "\n",
    "avg = avg_len(data_test)\n",
    "print(avg)\n",
    "avg_l = int(math.ceil(avg/4))\n",
    "print(avg_l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#10,11,12 UNIVERSAL TRUTH / FALSE / WIKIPEDIA LINES\n",
    "ut = pd.read_csv('universal truth.csv',encoding = 'utf-8', header = None ,names=['text'])\n",
    "uf = pd.read_csv('universal false.csv',encoding = 'utf-8',names=['text'])\n",
    "wiki =  pd.read_csv('wiki.csv',encoding = 'utf-8',names=['text'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('song.pickle', 'rb') as handle:\n",
    "    song= pickle.load(handle)\n",
    "\n",
    "song_lyric=[]\n",
    "for i in song:\n",
    "    song_lyric.append(tokenize.sent_tokenize(i))\n",
    "songs = [x for sublist in song_lyric for x in sublist]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Contractions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import shuffle\n",
    "\n",
    "def contractions(sentence):\n",
    "    split = nltk.sent_tokenize(sentence)\n",
    "    \n",
    "    contractions1 = {\n",
    "    \"ain't\": \"am not\",\n",
    "    \"aren't\": \"are not\",\n",
    "    \"can't\": \"cannot\",\n",
    "    \"can't've\": \"cannot have\",\n",
    "    \"'cause\": \"because\",\n",
    "    \"could've\": \"could have\",\n",
    "    \"couldn't\": \"could not\",\n",
    "    \"couldn't've\": \"could not have\",\n",
    "    \"didn't\": \"did not\",\n",
    "    \"doesn't\": \"does not\",\n",
    "    \"don't\": \"do not\",\n",
    "    \"hadn't\": \"had not\",\n",
    "    \"hadn't've\": \"had not have\",\n",
    "    \"hasn't\": \"has not\",\n",
    "    \"haven't\": \"have not\",\n",
    "    \"he'd\": \"he would\",\n",
    "    \"he'd've\": \"he would have\",\n",
    "    \"he'll\": \"he will\",\n",
    "    \"he'll've\": \"he will have\",\n",
    "    \"he's\": \"he is\",\n",
    "    \"how'd\": \"how did\",\n",
    "    \"how'd'y\": \"how do you\",\n",
    "    \"how'll\": \"how will\",\n",
    "    \"how's\": \"how is\",\n",
    "    \"i'd\": \"I would\",\n",
    "    \"i'd've\": \"I would have\",\n",
    "    \"i'll\": \"I will\",\n",
    "    \"i'll've\": \"I will have\",\n",
    "    \"i'm\": \"I am\",\n",
    "    \"i've\": \"I have\",\n",
    "    \"isn't\": \"is not\",\n",
    "    \"it'd\": \"it would\",\n",
    "    \"it'd've\": \"it would have\",\n",
    "    \"it'll\": \"it will\",\n",
    "    \"it'll've\": \"it will have\",\n",
    "    \"it's\": \"it is\",\n",
    "    \"let's\": \"let us\",\n",
    "    \"ma'am\": \"madam\",\n",
    "    \"mayn't\": \"may not\",\n",
    "    \"might've\": \"might have\",\n",
    "    \"mightn't\": \"might not\",\n",
    "    \"mightn't've\": \"might not have\",\n",
    "    \"must've\": \"must have\",\n",
    "    \"mustn't\": \"must not\",\n",
    "    \"mustn't've\": \"must not have\",\n",
    "    \"needn't\": \"need not\",\n",
    "    \"needn't've\": \"need not have\",\n",
    "    \"o'clock\": \"of the clock\",\n",
    "    \"oughtn't\": \"ought not\",\n",
    "    \"oughtn't've\": \"ought not have\",\n",
    "    \"shan't\": \"shall not\",\n",
    "    \"sha'n't\": \"shall not\",\n",
    "    \"shan't've\": \"shall not have\",\n",
    "    \"she'd\": \"she would\",\n",
    "    \"she'd've\": \"she would have\",\n",
    "    \"she'll\": \"she will\",\n",
    "    \"she'll've\": \"she will have\",\n",
    "    \"she's\": \"she is\",\n",
    "    \"should've\": \"should have\",\n",
    "    \"shouldn't\": \"should not\",\n",
    "    \"shouldn't've\": \"should not have\",\n",
    "    \"so've\": \"so have\",\n",
    "    \"so's\": \"so is\",\n",
    "    \"that'd\": \"that had\",\n",
    "    \"that'd've\": \"that would have\",\n",
    "    \"that's\": \"that is\",\n",
    "    \"there'd\": \"there would\",\n",
    "    \"there'd've\": \"there would have\",\n",
    "    \"there's\": \"there is\",\n",
    "    \"they'd\": \"they would\",\n",
    "    \"they'd've\": \"they would have\",\n",
    "    \"they'll\": \"they will\",\n",
    "    \"they'll've\": \"they will have\",\n",
    "    \"they're\": \"they are\",\n",
    "    \"they've\": \"they have\",\n",
    "    \"to've\": \"to have\",\n",
    "    \"wasn't\": \"was not\",\n",
    "    \"we'd\": \"we would\",\n",
    "    \"we'd've\": \"we would have\",\n",
    "    \"we'll\": \"we will\",\n",
    "    \"we'll've\": \"we will have\",\n",
    "    \"we're\": \"we are\",\n",
    "    \"we've\": \"we have\",\n",
    "    \"weren't\": \"were not\",\n",
    "    \"what'll\": \"what will\",\n",
    "    \"what'll've\": \" what will have\",\n",
    "    \"what're\": \"what are\",\n",
    "    \"what's\": \"what is\",\n",
    "    \"what've\": \"what have\",\n",
    "    \"when's\": \"when is\",\n",
    "    \"when've\": \"when have\",\n",
    "    \"where'd\": \"where did\",\n",
    "    \"where's\": \"where is\",\n",
    "    \"where've\": \"where have\",\n",
    "    \"who'll\": \"who will\",\n",
    "    \"who'll've\": \"who will have\",\n",
    "    \"who's\": \"who is\",\n",
    "    \"who've\": \"who have\",\n",
    "    \"why's\": \"why is\",\n",
    "    \"why've\": \"why have\",\n",
    "    \"will've\": \"will have\",\n",
    "    \"won't\": \"will not\",\n",
    "    \"won't've\": \"will not have\",\n",
    "    \"would've\": \"would have\",\n",
    "    \"wouldn't\": \"would not\",\n",
    "    \"wouldn't've\": \"would not have\",\n",
    "    \"y'all\": \"you all\",\n",
    "    \"y'all'd\": \"you all would\",\n",
    "    \"y'all'd've\": \"you all would have\",\n",
    "    \"y'all're\": \"you all are\",\n",
    "    \"y'all've\": \"you all have\",\n",
    "    \"you'd\": \"you would\",\n",
    "    \"you'd've\": \"you would have\",\n",
    "    \"you'll\": \"you will\",\n",
    "    \"you'll've\": \"you will have\",\n",
    "    \"you're\": \"you are\",\n",
    "    \"you've\": \"you have\"\n",
    "    }\n",
    "    inv_cont = {v: k for k, v in contractions1.items()}\n",
    " \n",
    "\n",
    "    res=[]\n",
    "    for k,v in inv_cont.items():\n",
    "        if(sentence.find(k)):\n",
    "            sentence = sentence.replace(k,v)\n",
    "\n",
    "    return sentence\n",
    "            \n",
    "            \n",
    "        \n",
    "\n",
    "    \n",
    "shu=[]\n",
    "for i,r in data_test.iterrows():\n",
    "    e = contractions(r['text'])\n",
    "    shu.append(e)\n",
    "\n",
    "shu_text = pd.DataFrame(shu,columns=['text']) \n",
    "shu_text.to_csv('contractions_aes_'+'prompt'+str(P)+'.csv',index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sentences from starting (one by one) ie (1st sentence, first 2 sentences .... first n sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Start Escalation \n",
    "\n",
    "def start_range(data,num_text,start_sen): \n",
    "\n",
    "    start=[]\n",
    "    text= nltk.sent_tokenize(data)\n",
    "\n",
    "    for i in range (len(text)):\n",
    "        num_text.append(i+1)\n",
    "        start_sen.append(''.join(text[:i+1]))\n",
    "        \n",
    "        \n",
    "        \n",
    "num_text =[]\n",
    "start_sen=[]\n",
    "for i,r in data_test.iterrows():\n",
    "    start_range(r['text'],num_text,start_sen)\n",
    "    \n",
    "start_escalation = pd.DataFrame() \n",
    "start_escalation['text'] = start_sen\n",
    "start_escalation['range'] = num_text\n",
    "\n",
    "start_escalation.to_csv('start_escalation_'+'prompt_'+str(P)+'.csv',index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sentences from end (one by one) ie (last sentence, last 2 sentences .... last n sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#End Esaclation\n",
    "def end_range(data,num_text,end_sen): \n",
    "\n",
    "    \n",
    "\n",
    "    text= list(reversed(nltk.sent_tokenize(data)))\n",
    "    for i in range(len(text)):\n",
    "        \n",
    "        num_text.append(i)\n",
    "        end_sen.append(''.join((text[:i+1])))\n",
    "        \n",
    "        \n",
    "num_text =[]\n",
    "end_sen=[]\n",
    "for i,r in data_test.iterrows():\n",
    "    end_range(r['text'],num_text,end_sen)\n",
    "end_escalation = pd.DataFrame() \n",
    "end_escalation['text'] = end_sen\n",
    "end_escalation['range'] = num_text   \n",
    "\n",
    "end_escalation.to_csv('end_escalation_'+'prompt_'+str(P)+'.csv',index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Adding song lyrics at stratigic position"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SONGS LYRICS ADDITION\n",
    "\n",
    "\n",
    "\n",
    "import random\n",
    "def insert_songs_beg(text):\n",
    "\n",
    "    \n",
    "    import random\n",
    "\n",
    "    ar = random.sample(songs, avg_l)\n",
    "    s = ''.join(ar)\n",
    "    res = s+'.'+text\n",
    "    return res\n",
    "\n",
    "def insert_songs_end(text):\n",
    "\n",
    "        \n",
    "    ar = random.sample(songs, avg_l)\n",
    "    s = ''.join(ar)\n",
    "    res = text+'.'+s\n",
    "    \n",
    "\n",
    "        \n",
    "    return res\n",
    "\n",
    "def insert_songs_mid(text):\n",
    "    text_data = tokenize.sent_tokenize(text)\n",
    "    ar = random.sample(songs, avg_l)\n",
    "    \n",
    "    for i in ar:\n",
    "        text_data.insert(int(len(text_data)/2), i)\n",
    "    return ''.join(text_data)\n",
    "\n",
    "songs_beg=[]\n",
    "for i,r in data_test.iterrows():\n",
    "    songs_beg.append(insert_songs_beg(r['text']))\n",
    "\n",
    "\n",
    "songs_text_beg =pd.DataFrame()\n",
    "songs_text_beg['text'] = songs_beg\n",
    "songs_text_beg.to_csv('songs_test_beg_'+'prompt_'+str(P)+'.csv',index=None)\n",
    "\n",
    "#---------------------------------------------------\n",
    "\n",
    "import numpy as np\n",
    "songs_end=[]\n",
    "for i,r in data_test.iterrows():\n",
    "    songs_end.append(insert_songs_end(r['text']))\n",
    "\n",
    "\n",
    "songs_text_end =pd.DataFrame()\n",
    "songs_text_end['text'] = songs_end\n",
    "songs_text_end.to_csv('songs_test_end_'+'prompt_'+str(P)+'.csv',index=None)\n",
    "\n",
    "#------------------------------------------------------\n",
    "\n",
    "songs_mid=[]\n",
    "for i,r in data_test.iterrows():\n",
    "    songs_mid.append(insert_songs_mid(r['text']))\n",
    "\n",
    "songs_text_mid =pd.DataFrame()\n",
    "songs_text_mid['text'] = songs_mid\n",
    "songs_text_mid.to_csv('songs_test_mid_'+'prompt_'+str(P)+'.csv',index=None)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Adding speeches in strategic positions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SPEECHES \n",
    "\n",
    "speech=[ 'My fellow Americans: Tonight, I am speaking to you because there is a growing humanitarian and security crisis at our southern border. Every day, Customs and Border Patrol agents encounter thousands of illegal immigrants trying to enter our country. We are out of space to hold them, and we have no way to promptly return them back home to their country. America proudly welcomes millions of lawful immigrants who enrich our society and contribute to our nation. But all Americans are hurt by uncontrolled, illegal migration. It strains public resources and drives down jobs and wages. Among those hardest hit are African Americans and Hispanic Americans. Our southern border is a pipeline for vast quantities of illegal drugs, including meth, heroin, cocaine, and fentanyl. Every week, 300 of our citizens are killed by heroin alone, 90 percent of which floods across from our southern border. More Americans will die from drugs this year than were killed in the entire Vietnam War. In the last two years, ICE officers made 266,000 arrests of aliens with criminal records, including those charged or convicted of 100,000 assaults, 30,000 sex crimes, and 4,000 violent killings. Over the years, thousands of Americans have been brutally killed by those who illegally entered our country, and thousands more lives will be lost if we dont act right now. This is a humanitarian crisis -- a crisis of the heart and a crisis of the soul.Last month, 20,000 migrant children were illegally brought into the United States -- a dramatic increase. These children are used as human pawns by vicious coyotes and ruthless gangs. One in three women are sexually assaulted on the dangerous trek up through Mexico. Women and children are the biggest victims, by far, of our broken system. This is the tragic reality of illegal immigration on our southern border. This is the cycle of human suffering that I am determined to end.Thank you so very much for being here. I love you all, too. Last night I congratulated Donald Trump and offered to work with him on behalf of our country.I hope that he will be a successful president for all Americans. This is not the outcome we wanted or we worked so hard for, and I am sorry we did not win this election for the values we share and the vision we hold for our country.But I feel pride and gratitude for this wonderful campaign that we built together. This vast, diverse, creative, unruly, energized campaign. You represent the best of America, and being your candidate has been one of the greatest honors of my life.I know how disappointed you feel, because I feel it too. And so do tens of millions of Americans who invested their hopes and dreams in this effort. This is painful, and it will be for a long time. But I want you to remember this.Our campaign was never about one person, or even one election. It was about the country we love and building an America that is hopeful, inclusive, and big-hearted. We have seen that our nation is more deeply divided than we thought. But I still believe in America, and I always will. And if you do, then we must accept this result and then look to the future. Donald Trump is going to be our president. We owe him an open mind and the chance to lead. Our constitutional democracy enshrines the peaceful transfer of power.We dont just respect that. We cherish it. It also enshrines the rule of law; the principle we are all equal in rights and dignity; freedom of worship and expression. We respect and cherish these values, too, and we must defend them.Let me add: Our constitutional democracy demands our participation, not just every four years, but all the time. So lets do all we can to keep advancing the causes and values we all hold dear. Making our economy work for everyone, not just those at the top, protecting our country and protecting our planet.And breaking down all the barriers that hold any American back from achieving their dreams. We spent a year and a half bringing together millions of people from every corner of our country to say with one voice that we believe that the American dream is big enough for everyone.For people of all races, and religions, for men and women, for immigrants, for LGBT people, and people with disabilities. For everyone.I am so grateful to stand with all of you. I want to thank Tim Kaine and Anne Holton for being our partners on this journey.We were promised compassionate conservatism and all we got was Katrina and wiretaps. We were promised a uniter, and we got a President who could not even lead the half of the country that voted for him. We were promised a more ethical and more efficient government,and instead we have a town called Washington that is more corrupt and more wasteful than it was before. And the only mission that was ever accomplished is to use fear and falsehood to take this country to a war that should have never been authorized and should have never been waged.My fellow Americans: Tonight, I am speaking to you because there is a growing humanitarian and security crisis at our southern border. Every day, Customs and Border Patrol agents encounter thousands of illegal immigrants trying to enter our country. We are out of space to hold them, and we have no way to promptly return them back home to their country. America proudly welcomes millions of lawful immigrants who enrich our society and contribute to our nation. But all Americans are hurt by uncontrolled, illegal migration. It strains public resources and drives down jobs and wages. Among those hardest hit are African Americans and Hispanic Americans. Our southern border is a pipeline for vast quantities of illegal drugs, including meth, heroin, cocaine, and fentanyl. Every week, 300 of our citizens are killed by heroin alone, 90 percent of which floods across from our southern border. More Americans will die from drugs this year than were killed in the entire Vietnam War. In the last two years, ICE officers made 266,000 arrests of aliens with criminal records, including those charged or convicted of 100,000 assaults, 30,000 sex crimes, and 4,000 violent killings. Over the years, thousands of Americans have been brutally killed by those who illegally entered our country, and thousands more lives will be lost if we dont act right now. This is a humanitarian crisis -- a crisis of the heart and a crisis of the soul.Last month, 20,000 migrant children were illegally brought into the United States -- a dramatic increase. These children are used as human pawns by vicious coyotes and ruthless gangs. One in three women are sexually assaulted on the dangerous trek up through Mexico. Women and children are the biggest victims, by far, of our broken system. This is the tragic reality of illegal immigration on our southern border. This is the cycle of human suffering that I am determined to end.Thank you so very much for being here. I love you all, too. Last night I congratulated Donald Trump and offered to work with him on behalf of our country.I hope that he will be a successful president for all Americans. This is not the outcome we wanted or we worked so hard for, and I am sorry we did not win this election for the values we share and the vision we hold for our country.But I feel pride and gratitude for this wonderful campaign that we built together. This vast, diverse, creative, unruly, energized campaign. You represent the best of America, and being your candidate has been one of the greatest honors of my life.I know how disappointed you feel, because I feel it too. And so do tens of millions of Americans who invested their hopes and dreams in this effort. This is painful, and it will be for a long time. But I want you to remember this.Our campaign was never about one person, or even one election. It was about the country we love and building an America that is hopeful, inclusive, and big-hearted. We have seen that our nation is more deeply divided than we thought. But I still believe in America, and I always will. And if you do, then we must accept this result and then look to the future. Donald Trump is going to be our president. We owe him an open mind and the chance to lead. Our constitutional democracy enshrines the peaceful transfer of power.We dont just respect that. We cherish it. It also enshrines the rule of law; the principle we are all equal in rights and dignity; freedom of worship and expression. We respect and cherish these values, too, and we must defend them.Let me add: Our constitutional democracy demands our participation, not just every four years, but all the time. So lets do all we can to keep advancing the causes and values we all hold dear. Making our economy work for everyone, not just those at the top, protecting our country and protecting our planet.And breaking down all the barriers that hold any American back from achieving their dreams. We spent a year and a half bringing together millions of people from every corner of our country to say with one voice that we believe that the American dream is big enough for everyone.For people of all races, and religions, for men and women, for immigrants, for LGBT people, and people with disabilities. For everyone.I am so grateful to stand with all of you. I want to thank Tim Kaine and Anne Holton for being our partners on this journey.We were promised compassionate conservatism and all we got was Katrina and wiretaps. We were promised a uniter, and we got a President who could not even lead the half of the country that voted for him. We were promised a more ethical and more efficient government,and instead we have a town called Washington that is more corrupt and more wasteful than it was before. And the only mission that was ever accomplished is to use fear and falsehood to take this country to a war that should have never been authorized and should have never been waged.']\n",
    "from nltk import tokenize\n",
    "speeches=[]\n",
    "for i in speech:\n",
    "    speeches.append(tokenize.sent_tokenize(i))\n",
    "speeches=speeches[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#4\n",
    "def insert_speeches_beg(text):\n",
    "\n",
    "    import random\n",
    "\n",
    "    ar = random.sample(speeches, avg_l) \n",
    "    s = ''.join(ar)\n",
    "    res = s+'.'+text\n",
    "        \n",
    "    return res\n",
    "\n",
    "\n",
    "\n",
    "def insert_speeches_end(text):\n",
    "\n",
    "    ar = random.sample(speeches, avg_l)\n",
    "    s = ''.join(ar)\n",
    "    res = text+'.'+s\n",
    "    \n",
    "\n",
    "    return res\n",
    "\n",
    "def insert_speeches_mid(text):\n",
    "\n",
    "    text_data = tokenize.sent_tokenize(text)\n",
    "    \n",
    "    ar = random.sample(speeches, avg_l)\n",
    "    for i in ar:\n",
    "        text_data.insert(int(len(text_data)/2), i)\n",
    "    return ''.join(text_data)\n",
    "\n",
    "\n",
    "speech=[]\n",
    "for i,r in data_test.iterrows():\n",
    "    speech.append(insert_speeches_beg(r['text']))\n",
    "\n",
    "\n",
    "speech_text_beg =pd.DataFrame()\n",
    "speech_text_beg['text'] = speech\n",
    "\n",
    "speech_text_beg.to_csv('speeches_test_beg_'+'prompt_'+str(P)+'.csv',index=None)\n",
    "\n",
    "\n",
    "speech_mid=[]\n",
    "for i,r in data_test.iterrows():\n",
    "    speech_mid.append(insert_speeches_mid(r['text']))\n",
    "\n",
    "\n",
    "speech_text_mid =pd.DataFrame()\n",
    "speech_text_mid['text'] = speech_mid\n",
    "\n",
    "speech_text_mid.to_csv('speeches_test_mid_'+'prompt_'+str(P)+'.csv',index=None)\n",
    "\n",
    "\n",
    "speech_end=[]\n",
    "for i,r in data_test.iterrows():\n",
    "    speech_end.append(insert_speeches_end(r['text']))\n",
    "\n",
    "\n",
    "speech_text_end =pd.DataFrame()\n",
    "speech_text_end['text'] = speech_end\n",
    "\n",
    "speech_text_end.to_csv('speeches_test_end_'+'prompt_'+str(P)+'.csv',index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Repitition of lines  (1/2/3) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5 REPEAT\n",
    "\n",
    "#repeat a line from beginning in the conclusion\n",
    "def random_repeat_conclusion_1(text):\n",
    "    text1 = text\n",
    "    text=tokenize.sent_tokenize(text)\n",
    "    \n",
    "    import random\n",
    "\n",
    "    if(len(text)>1):\n",
    "        ss=text[0]\n",
    "        s= ''.join(text) + '.' + ss\n",
    "    else : \n",
    "        s = text1+text1\n",
    "    return s\n",
    "\n",
    "#repeat 2 lines from beginning in the conclusion\n",
    "def random_repeat_conclusion_2(text):\n",
    "    text1 = text\n",
    "    text=tokenize.sent_tokenize(text)\n",
    "    \n",
    "    import random\n",
    "\n",
    "    if(len(text)>1):\n",
    "        ss=text[0]+text[1]\n",
    "        s= ''.join(text) + '.' + ss\n",
    "    else : \n",
    "        s = text1+text1\n",
    "    return s\n",
    "\n",
    "#repeat 1 line from conclusion in the beginning\n",
    "def random_repeat_introduction_1(text):\n",
    "    text1 =text\n",
    "    text=tokenize.sent_tokenize(text)\n",
    "    import random\n",
    "\n",
    "    if(len(text)>1):\n",
    "        ss=text[-1]\n",
    "        s= ss+'.'+''.join(text)\n",
    "    else : \n",
    "        s = text1+text1\n",
    "    return s\n",
    "\n",
    "#repeat 2 lines from conclusion in the beginning \n",
    "def random_repeat_introduction_2(text):\n",
    "    text1 = text\n",
    "    text=tokenize.sent_tokenize(text)\n",
    "    \n",
    "    import random\n",
    "\n",
    "    if(len(text)>1):\n",
    "        ss=text[-2]+text[-1]\n",
    "        s= ss+'.'+''.join(text)\n",
    "    else : \n",
    "        s = text1+text1\n",
    "    return s\n",
    "\n",
    "#repeat 1 line in middle \n",
    "def random_repeat_middle_1(text):\n",
    "    text1 =text\n",
    "    text=tokenize.sent_tokenize(text)\n",
    "    \n",
    "    import random\n",
    "\n",
    "    if(len(text)>1):\n",
    "        \n",
    "        ss=text[int(len(text)/2)]\n",
    "        text.insert(int(len(text)/2),ss)\n",
    "        s = ''.join(text)\n",
    "    else:\n",
    "        s= text1+text1\n",
    "    \n",
    "        \n",
    "    \n",
    "    \n",
    "    return s\n",
    "\n",
    "#repeat 2 lines in middle\n",
    "\n",
    "def random_repeat_middle_2(text):\n",
    "    text1=text\n",
    "    text=tokenize.sent_tokenize(text)\n",
    "    \n",
    "    import random\n",
    "    if(len(text)>=2):\n",
    "        \n",
    "        ss=text[int(len(text)/2)-1]+text[int(len(text)/2)]\n",
    "        text.insert(int(len(text)/2)-1,ss)\n",
    "        s = ''.join(text)\n",
    "    else:\n",
    "        s= text1+text1\n",
    "    \n",
    "    \n",
    "        \n",
    "    return s\n",
    "\n",
    "#repeat 3 lines in middle\n",
    "def random_repeat_middle_3(text):\n",
    "    text1 = text\n",
    "    text=tokenize.sent_tokenize(text)\n",
    "    \n",
    "    import random\n",
    "    if(len(text)>=3):\n",
    "        \n",
    "        ss=text[int(len(text)/2)-1]+text[int(len(text)/2)]+text[int(len(text)/2)+1]\n",
    "        text.insert(int(len(text)/2)-1,ss)\n",
    "        s = ''.join(text)\n",
    "        \n",
    "    else:\n",
    "#         t= text.append(text)\n",
    "        s = text1+text1\n",
    "        \n",
    "    return s\n",
    "\n",
    "fun_name =[random_repeat_conclusion_1,random_repeat_conclusion_2,random_repeat_introduction_1,random_repeat_introduction_2,random_repeat_middle_1,random_repeat_middle_2,random_repeat_middle_3]\n",
    "fun_name_1 =['repeat_test_conc1','repeat_test_conc2','repeat_test_into1','repeat_test_into2','repeat_test_middle1','repeat_test_middle2','repeat_test_middle3']\n",
    "def total_repeat(fun_name):\n",
    "    n=0\n",
    "    for k in fun_name:\n",
    "        \n",
    "        rand_rep=[]\n",
    "        for i,r in data_test.iterrows():\n",
    "            rand_rep.append(k(r['text']))\n",
    "\n",
    "        repeat_test =pd.DataFrame()\n",
    "        repeat_test['text'] = rand_rep\n",
    "        print (repeat_test.head())\n",
    "        repeat_test.to_csv(fun_name_1[n]+'_'+'prompt_'+str(P)+'.csv',index=None)\n",
    "        n=n+1\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                text\n",
      "0    A long time ago when I was in third grade I ...\n",
      "1   I believe that with all people laughter, and ...\n",
      "2   People always say laughter is a big part in a...\n",
      "3                 \"Laughter\" A good relationship ...\n",
      "4   Everyone enjoys laughter.Everywhere you look ...\n",
      "                                                text\n",
      "0    A long time ago when I was in third grade I ...\n",
      "1   I believe that with all people laughter, and ...\n",
      "2   People always say laughter is a big part in a...\n",
      "3                 \"Laughter\" A good relationship ...\n",
      "4   Everyone enjoys laughter.Everywhere you look ...\n",
      "                                                text\n",
      "0  FIN.  A long time ago when I was in third grad...\n",
      "1  It's no life at all, for laughter will always ...\n",
      "2  That's @CAPS2 makes our family relationship st...\n",
      "3  I've truly realized the meaning of the saying,...\n",
      "4  What makes you laugh and smile?. Everyone enjo...\n",
      "                                                text\n",
      "0  She said @CAPS1 but laughter isn't going to he...\n",
      "1  What is a life without a diamond in the rough ...\n",
      "2  I can never be mad at my grandparents because ...\n",
      "3  Our bond, our way of getting over tough times,...\n",
      "4  My niece and nephew is just one example.What m...\n",
      "                                                text\n",
      "0    A long time ago when I was in third grade I ...\n",
      "1   I believe that with all people laughter, and ...\n",
      "2   People always say laughter is a big part in a...\n",
      "3                 \"Laughter\" A good relationship ...\n",
      "4   Everyone enjoys laughter.Everywhere you look ...\n",
      "                                                text\n",
      "0    A long time ago when I was in third grade I ...\n",
      "1   I believe that with all people laughter, and ...\n",
      "2   People always say laughter is a big part in a...\n",
      "3                 \"Laughter\" A good relationship ...\n",
      "4   Everyone enjoys laughter.Everywhere you look ...\n",
      "                                                text\n",
      "0    A long time ago when I was in third grade I ...\n",
      "1   I believe that with all people laughter, and ...\n",
      "2   People always say laughter is a big part in a...\n",
      "3                 \"Laughter\" A good relationship ...\n",
      "4   Everyone enjoys laughter.Everywhere you look ...\n"
     ]
    }
   ],
   "source": [
    "total_repeat(fun_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove 25-50% of lines from response from any position"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#6REMOVE\n",
    "def random_remove(text):\n",
    "    current=tokenize.sent_tokenize(text)\n",
    "    \n",
    "    if(len(current) >1):\n",
    "        import random\n",
    "        num =int(avg_l)\n",
    "        while(num >len(current)):\n",
    "            num =random.randint(1,int(avg_l))\n",
    "\n",
    "        ar = random.sample(current, num)\n",
    "        fin = (x for x in current if x not in set(ar))\n",
    "        res= ''.join(fin)\n",
    "        \n",
    "        if(res==''):\n",
    "            res=''.join(current)\n",
    "    else: \n",
    "        res = ''.join(current)\n",
    "    return res\n",
    "    \n",
    "rand_rem=[]\n",
    "for i,r in data_test.iterrows():\n",
    "    rand_rem.append(random_remove(r['text']))\n",
    "    \n",
    "rem_text = pd.DataFrame(rand_rem,columns=['text']) \n",
    "rem_text.to_csv('remove_test_'+'prompt_'+str(P)+'.csv',index=None)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Shuffling the sentences in a response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#7SHUFFLE\n",
    "\n",
    "from random import shuffle\n",
    "\n",
    "def scramble(sentence):\n",
    "    split = tokenize.sent_tokenize(sentence)\n",
    "    shuffle(split)  # This shuffles the list in-place.\n",
    "    return ''.join(split)  # Turn the list back into a string\n",
    "shu=[]\n",
    "for i,r in data_test.iterrows():\n",
    "    e = scramble(r['text'])\n",
    "    shu.append(e)\n",
    "shu_text = pd.DataFrame(shu,columns=['text']) \n",
    "shu_text.to_csv('shuffle_test_'+'prompt_'+str(P)+'.csv',index=None)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Incorrect grammar and disfluency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#8,9 INCORRECT GRAMMAR , DISFLUENCY\n",
    "import random\n",
    "def ig_frequency(data_test,avg):\n",
    "\n",
    "    l= frequency(data_test,avg)\n",
    "    l.to_csv('disfluency_'+str(P)+'.csv',index=None)\n",
    "    c= incorrect_grammar(data_test)\n",
    "    c.to_csv('incorrect_grammar_'+str(P)+'.csv',index=None)\n",
    "    \n",
    "\n",
    "    \n",
    "def frequency(data,avg_l):\n",
    "    from nltk import sent_tokenize\n",
    "    arr =[]\n",
    "    dis=[ \"huh\", \"uh..\", \"erm\", \"um\", \"well..\", \"so..\", \"like\", \"hmm\",\"I mean ...\",\"Ahhh\"]\n",
    "    for i,r in data.iterrows():\n",
    "        s=sent_tokenize(r['text'])\n",
    "        for j in range(int((avg_l))):\n",
    "            r = random.randint(0,len(s)-1)\n",
    "            l = random.choice(dis) + ' ' +s[r]\n",
    "            s[r] = l\n",
    "        res = ''.join(s)\n",
    "        arr.append(res)\n",
    "    \n",
    "    fin =pd.DataFrame()\n",
    "    fin['text'] = arr\n",
    "    return fin\n",
    "        \n",
    "        \n",
    "    \n",
    "def incorrect_grammar(data):\n",
    "    arr=[]\n",
    "    for i,r in data.iterrows():\n",
    "        s = str(r['text'].lower())\n",
    "        s =s.replace('a ', 'an ')\n",
    "        s = s.replace('the','a')\n",
    "        s= s.replace('would','will')\n",
    "        s= s.replace('\\'s','s')\n",
    "        s=s.replace('should','will')\n",
    "        s=s.replace('was','should')\n",
    "        s = s.replace('es','e')\n",
    "        s= s.replace('this','that')\n",
    "        s=s.replace('is','in')\n",
    "        s=s.replace('are','is')\n",
    "        s=s.replace('have','had')\n",
    "        s=s.replace('their','there')\n",
    "        s=s.replace('of','on')\n",
    "        s= s.replace ('i ','we ')\n",
    "        s.replace('.',',')\n",
    "        s= s.replace('to ','2 ')\n",
    "        s= s.replace('for','from')\n",
    "        s=s.replace('has','have')\n",
    "        s=s.replace('you','u')\n",
    "        s=s.replace('\\'',' ')\n",
    "        s=s.replace('and','&')\n",
    "\n",
    "        arr.append(s)\n",
    "    fin =pd.DataFrame()\n",
    "    fin['text'] = arr\n",
    "    return fin\n",
    "\n",
    "ig_frequency(data_test,avg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Addition of universal truth lines, universal false lines, wikipedia lines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Position of adding and change in number of lines of response\n",
    "bounded : length of the respose doesnt change\n",
    "\n",
    "unbounded : length of the response changes \n",
    "\n",
    "beg : insert in beginning \n",
    "\n",
    "mid : insert in middle\n",
    "\n",
    "end : insert in end "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### For wikipedia articles , find the best topics from each prompt (topic related):\n",
    "    Topic Specifc : from the top topic find the related article (wikipedia) and add lines from article to responses.\n",
    "    Related Lines : from the few other topics , find the related wikipedia articles, and add these lines to the reponse.\n",
    "    Not specific  : from any wikipedia articles , extracted lines and add them to the response."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import tokenize\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#BASE FUNCTIONS FOR ADDITION UNBOUNDED\n",
    "def insert_beg(text,avg_l,sim):\n",
    "    \n",
    "\n",
    "    res=[]\n",
    "    for text in text:\n",
    "    \n",
    "        import random\n",
    "        ar = random.sample(sim, avg_l)\n",
    "        s = ''.join(str(e) for e in ar)\n",
    "\n",
    "        res.append(s+'.'+text)\n",
    "  \n",
    "        \n",
    "    return res\n",
    "\n",
    "\n",
    "def insert_middle(text,avg_l,sim):\n",
    "\n",
    "    res=[]\n",
    "    for text in text:\n",
    "        text = tokenize.sent_tokenize(text)\n",
    "        import random\n",
    "        ar = random.sample(sim, avg_l)\n",
    "        ss = '.'.join(str(e) for e in ar)\n",
    "        \n",
    "        if(len(text)>1):\n",
    "\n",
    "            text.insert(int(len(text)/2),ss)\n",
    "        s = ''.join(text)\n",
    "        res.append(s)\n",
    "    return res\n",
    "\n",
    "\n",
    "def insert_end(text,avg_l,sim):\n",
    "    res=[]\n",
    "    for text in text:    \n",
    "        ar = random.sample(sim, avg_l)\n",
    "\n",
    "        s = ''.join(str(e) for e in ar)\n",
    "        res.append(text+'.'+s)\n",
    "\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#BASE FUNCTIONS FOR ADDITION BOUNDED\n",
    "def insert_beg_bound(text,avg_l,sim):\n",
    "    \n",
    "    res=[]\n",
    "    for text in text:\n",
    "        avg=int(avg_l)\n",
    "        import random\n",
    "        import nltk\n",
    "        \n",
    "        curr= tokenize.sent_tokenize(text)\n",
    "        if(len(tokenize.sent_tokenize(text)) < avg_l):\n",
    "            avg = max(1,int(len(tokenize.sent_tokenize(text))/2))\n",
    "\n",
    "\n",
    "        new_curr = curr[:len(curr) - avg]\n",
    "\n",
    "        ar = random.sample(sim, avg)\n",
    "        s = ''.join(str(e) for e in ar)\n",
    "#         \n",
    "        text = ''.join(str(e) for e in new_curr)\n",
    "        res.append( s+'.'+text)\n",
    "#     print(len(res))\n",
    "        \n",
    "    return res\n",
    "\n",
    "\n",
    "def insert_middle_bound(text,avg_l,sim):\n",
    "    \n",
    "    res=[]\n",
    "    for text in text:\n",
    "        avg=int(avg_l)\n",
    "        curr= tokenize.sent_tokenize(text)\n",
    "        if(len(tokenize.sent_tokenize(text)) < avg_l):\n",
    "            avg= max(1,int(len(tokenize.sent_tokenize(text))/2))\n",
    "\n",
    "        new_curr = curr[:len(curr) - avg]\n",
    "\n",
    "        import random\n",
    "        ar = random.sample(sim, avg)\n",
    "        ss = ''.join(str(e) for e in ar)\n",
    "\n",
    "        if(len(text)>1):\n",
    "\n",
    "            curr.insert(int(len(curr)/2),ss)\n",
    "        ss = ''.join(curr)\n",
    "        res.append(ss)\n",
    "    return res\n",
    "\n",
    "\n",
    "def insert_end_bound(text,avg_l,sim):\n",
    "#     current=fin.split('.')\n",
    "    res=[]\n",
    "    for text in text:\n",
    "        avg=int(avg_l)\n",
    "        curr=tokenize.sent_tokenize(text)\n",
    "        if(len(tokenize.sent_tokenize(text)) < avg_l):\n",
    "            avg = max(1,int(len(tokenize.sent_tokenize(text))/2))\n",
    "\n",
    "\n",
    "        new_curr = curr[:len(curr) - avg]\n",
    "        text = ''.join(str(e) for e in new_curr)\n",
    "\n",
    "        ar = random.sample(sim, avg)\n",
    "\n",
    "        s = ''.join(str(e) for e in ar)\n",
    "        res.append(text+'.'+s)\n",
    "#     print(len(res))\n",
    "        \n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def insert_text(text,avg_l,sample,num,prompt):\n",
    "    \n",
    "    beg_unbound = insert_beg(text,avg_l,sample)\n",
    "    mid_unbound= insert_middle(text,avg_l,sample)\n",
    "    end_unbound= insert_end(text,avg_l,sample)\n",
    "    beg_bound = insert_beg_bound(text,avg_l,sample)\n",
    "    mid_bound=insert_middle_bound(text,avg_l,sample)\n",
    "    end_bound= insert_end_bound(text,avg_l,sample)\n",
    "    tot = [beg_unbound ,\n",
    "    mid_unbound,\n",
    "    end_unbound,\n",
    "    beg_bound ,\n",
    "    mid_bound,\n",
    "    end_bound]\n",
    "    sample_names=['ut','uf','wiki']#,'wiki_topic','wiki_rel']\n",
    "    tot_names= ['beg_unbound' ,\n",
    "    'mid_unbound',\n",
    "    'end_unbound',\n",
    "    'beg_bound' ,\n",
    "    'mid_bound',\n",
    "    'end_bound']\n",
    "    for i in range(len(tot)):\n",
    "        \n",
    "        df = pd.DataFrame()\n",
    "        df['text'] = tot[i]\n",
    "        df.to_csv(sample_names[num]+'_'+tot_names[i]+'_'+ str(prompt)+'.csv',index=None)\n",
    "    return tot\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "al = int(avg_l)\n",
    "sample = [ut,uf,wiki]#,wiki_topic,wiki_rel]\n",
    "text = data_test['text']\n",
    "num=0\n",
    "for j in sample:\n",
    "\n",
    "    j=j['text'].values.tolist()\n",
    "    t = insert_text(text,al,j,num,P)\n",
    "    num=num+1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Addition of entities into the response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#13 ENTITIES ADDITION\n",
    "\n",
    "# entities = pd.read_csv('entities.csv')\n",
    "# sim=entities['text'].values.tolist()\n",
    "sim=wiki['text'][1:].tolist()\n",
    "def insert_entities(text,avg_l,prompt,sample):\n",
    "    \n",
    "    beg_unbound = insert_beg(text,avg_l,sample)\n",
    "    mid_unbound= insert_middle(text,avg_l,sample)\n",
    "    end_unbound= insert_end(text,avg_l,sample)\n",
    "    beg_bound = insert_beg_bound(text,avg_l,sample)\n",
    "    mid_bound=insert_middle_bound(text,avg_l,sample)\n",
    "    end_bound= insert_end_bound(text,avg_l,sample)\n",
    "    \n",
    "    tot = [beg_unbound ,\n",
    "    mid_unbound,\n",
    "    end_unbound,\n",
    "    beg_bound ,\n",
    "    mid_bound,\n",
    "    end_bound]\n",
    "    \n",
    "    tot_names= ['beg_unbound' ,\n",
    "    'mid_unbound',\n",
    "    'end_unbound',\n",
    "    'beg_bound' ,\n",
    "    'mid_bound',\n",
    "    'end_bound']\n",
    "    \n",
    "    for i in range(len(tot)):\n",
    "        \n",
    "        df = pd.DataFrame()\n",
    "        df['text'] = tot[i]\n",
    "        df.to_csv('entities'+'_'+tot_names[i]+'_'+ str(prompt)+'.csv',index=None)\n",
    "        \n",
    "    return tot\n",
    "\n",
    "text = data_test['text']\n",
    "al = int(avg)\n",
    "prompt =P\n",
    "rc =insert_entities(text,al,prompt,sim)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Adding synonyms of particular word in the sentence in its place"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#PARAPHRASHING \n",
    "\n",
    "from itertools import chain\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import wordpunct_tokenize\n",
    "import random\n",
    "stop_words = set(stopwords.words('english'))\n",
    "stop_words.update(['.', ',', '\"', \"'\", '?', '!', ':', ';', '(', ')', '[', ']', '{', '}'])\n",
    "\n",
    "def paraphrase(data_test):\n",
    "    ft=[]\n",
    "    for i,r in data_test.iterrows():\n",
    "        \n",
    "        text =tokenize.sent_tokenize(r['text'])\n",
    "        sent=[]\n",
    "        for t in text:\n",
    "            \n",
    "\n",
    "            list_of_words = [i.lower() for i in wordpunct_tokenize(t) if i.lower() not in stop_words]\n",
    "            low = list_of_words.copy()\n",
    "            if(len(list_of_words) != 0):\n",
    "                \n",
    "                if(len(list_of_words)>1):\n",
    "                    n = max(1,random.randint(2,len(list_of_words)))\n",
    "                else:\n",
    "                    n=1\n",
    "                for k in range(n):\n",
    "                    ind = random.randint(0,len(list_of_words)-1) \n",
    "                    w = list_of_words[ind]\n",
    "\n",
    "                    synonyms = wordnet.synsets(w)\n",
    "                    lemmas = list(set(chain.from_iterable([word.lemma_names() for word in synonyms])))\n",
    "\n",
    "                    if(len(lemmas) != 0):\n",
    "                        low[ind] = lemmas[0]\n",
    "                sent.append(' '.join(low))\n",
    "\n",
    "            else:\n",
    "                sent.append(t)\n",
    "        ft.append('.'.join(sent))\n",
    "    df_para =pd.DataFrame()\n",
    "    df_para['text']= ft\n",
    "    df_para.to_csv('para_P_'+str(P)+'.csv',index=None)\n",
    "paraphrase(data_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}