TF-IDF_ASDiv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "364275aa-456e-4700-a14d-863a0851d209",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_json('ASDiv_instruct.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "8c687a80-ea4b-4c83-8a89-7a4a319e1acf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>instruction</th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Write a grade 1 math word problem.</td>\n",
       "      <td></td>\n",
       "      <td>Seven red apples and two green apples are in t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Write a grade 1 math word problem.</td>\n",
       "      <td></td>\n",
       "      <td>Ellen has six more balls than Marin. Marin has...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Write a grade 1 math word problem.</td>\n",
       "      <td></td>\n",
       "      <td>Janet has nine oranges and Sharon has seven or...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Write a grade 1 math word problem.</td>\n",
       "      <td></td>\n",
       "      <td>Allan brought two balloons and Jake brought fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Write a grade 1 math word problem.</td>\n",
       "      <td></td>\n",
       "      <td>Adam has five more apples than Jackie. Jackie ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2300</th>\n",
       "      <td>Write a grade 4 math word problem.</td>\n",
       "      <td></td>\n",
       "      <td>There are 3300 bananas in Janice's banana coll...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2301</th>\n",
       "      <td>Write a grade 4 math word problem.</td>\n",
       "      <td></td>\n",
       "      <td>Janice has 316 bottle caps that must be put aw...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2302</th>\n",
       "      <td>Write a grade 4 math word problem.</td>\n",
       "      <td></td>\n",
       "      <td>Chris is inviting 82 friends to a party. He ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2303</th>\n",
       "      <td>Write a grade 4 math word problem.</td>\n",
       "      <td></td>\n",
       "      <td>Keith has 5530 marbles and 3 pencils. If he sh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2304</th>\n",
       "      <td>Write a grade 4 math word problem.</td>\n",
       "      <td></td>\n",
       "      <td>There are 43 students and 1720 apples. Each st...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2305 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             instruction input  \\\n",
       "0     Write a grade 1 math word problem.         \n",
       "1     Write a grade 1 math word problem.         \n",
       "2     Write a grade 1 math word problem.         \n",
       "3     Write a grade 1 math word problem.         \n",
       "4     Write a grade 1 math word problem.         \n",
       "...                                  ...   ...   \n",
       "2300  Write a grade 4 math word problem.         \n",
       "2301  Write a grade 4 math word problem.         \n",
       "2302  Write a grade 4 math word problem.         \n",
       "2303  Write a grade 4 math word problem.         \n",
       "2304  Write a grade 4 math word problem.         \n",
       "\n",
       "                                                 output  \n",
       "0     Seven red apples and two green apples are in t...  \n",
       "1     Ellen has six more balls than Marin. Marin has...  \n",
       "2     Janet has nine oranges and Sharon has seven or...  \n",
       "3     Allan brought two balloons and Jake brought fo...  \n",
       "4     Adam has five more apples than Jackie. Jackie ...  \n",
       "...                                                 ...  \n",
       "2300  There are 3300 bananas in Janice's banana coll...  \n",
       "2301  Janice has 316 bottle caps that must be put aw...  \n",
       "2302  Chris is inviting 82 friends to a party. He ha...  \n",
       "2303  Keith has 5530 marbles and 3 pencils. If he sh...  \n",
       "2304  There are 43 students and 1720 apples. Each st...  \n",
       "\n",
       "[2305 rows x 3 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "14188271-8472-44c3-86f3-1c32890bee92",
   "metadata": {},
   "outputs": [],
   "source": [
    "grades = ['1', '2', '3', '4', '5', '6']\n",
    "prompt = f\"Write a grade {grade} math word problem.\"\n",
    "grade1 = []\n",
    "grade2 = []\n",
    "grade3 = []\n",
    "grade4 = []\n",
    "grade5 = []\n",
    "grade6 = []\n",
    "for grade in grades:\n",
    "    questions = df.query(f\"instruction=='Write a grade {grade} math word problem.'\")['output']\n",
    "    if grade == '1':\n",
    "        for question in questions:\n",
    "            grade1.append(question)\n",
    "    if grade == '2':\n",
    "        for question in questions:\n",
    "            grade2.append(question)\n",
    "            \n",
    "    if grade == '3':\n",
    "        for question in questions:\n",
    "            grade3.append(question)\n",
    "            \n",
    "    if grade == '4':\n",
    "        for question in questions:\n",
    "            grade4.append(question)\n",
    "            \n",
    "    if grade == '5':\n",
    "        for question in questions:\n",
    "            grade5.append(question)\n",
    "            \n",
    "    if grade == '6':\n",
    "        for question in questions:\n",
    "            grade6.append(question)\n",
    "\n",
    "grade1 = \"\\n\".join(grade1)\n",
    "grade2 = \"\\n\".join(grade2)\n",
    "grade3 = \"\\n\".join(grade3)\n",
    "grade4 = \"\\n\".join(grade4)\n",
    "grade5 = \"\\n\".join(grade5)\n",
    "grade6 = \"\\n\".join(grade6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "3997ef7f-80f2-47a8-af0b-b5f6aecfdc40",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import pandas as pd\n",
    "pd.set_option(\"max_rows\", 600)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "1824a489-1aef-4ffa-9d5b-b98d8fa3bb79",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = [grade1, grade2, grade3, grade4, grade5, grade6]\n",
    "#tfidf_vectorizer = TfidfVectorizer(ngram_range = (3,3), stop_words='english')\n",
    "tfidf_vectorizer = TfidfVectorizer(ngram_range = (1,3))\n",
    "tfidf_vector = tfidf_vectorizer.fit_transform(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "f64ff343-5308-4d86-accb-980a6549476c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "88944"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tfidf_vectorizer.idf_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "fa81a64d-79c2-416e-bb95-c93bcce79ca1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>00</th>\n",
       "      <th>00 and</th>\n",
       "      <th>00 and all</th>\n",
       "      <th>00 and he</th>\n",
       "      <th>00 and they</th>\n",
       "      <th>00 before</th>\n",
       "      <th>00 before how</th>\n",
       "      <th>00 buying</th>\n",
       "      <th>00 buying gifts</th>\n",
       "      <th>00 charlie</th>\n",
       "      <th>...</th>\n",
       "      <th>zoo they sit</th>\n",
       "      <th>zoo three</th>\n",
       "      <th>zoo three times</th>\n",
       "      <th>zoo tickets</th>\n",
       "      <th>zoo tickets cost</th>\n",
       "      <th>zoo zoe</th>\n",
       "      <th>zoo zoe took</th>\n",
       "      <th>zookeeper</th>\n",
       "      <th>zookeeper has</th>\n",
       "      <th>zookeeper has 68</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.010226</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002462</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002462</td>\n",
       "      <td>0.002462</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.029610</td>\n",
       "      <td>0.003503</td>\n",
       "      <td>0.000876</td>\n",
       "      <td>0.000876</td>\n",
       "      <td>0.001751</td>\n",
       "      <td>0.000876</td>\n",
       "      <td>0.000876</td>\n",
       "      <td>0.000876</td>\n",
       "      <td>0.000876</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000876</td>\n",
       "      <td>0.000876</td>\n",
       "      <td>0.000876</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000876</td>\n",
       "      <td>0.000876</td>\n",
       "      <td>0.000876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002568</td>\n",
       "      <td>0.002568</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.003788</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.003137</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6 rows × 88944 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         00    00 and  00 and all  00 and he  00 and they  00 before  \\\n",
       "0  0.000000  0.000000    0.000000   0.000000     0.000000   0.000000   \n",
       "1  0.010226  0.000000    0.000000   0.000000     0.000000   0.000000   \n",
       "2  0.029610  0.003503    0.000876   0.000876     0.001751   0.000876   \n",
       "3  0.000000  0.000000    0.000000   0.000000     0.000000   0.000000   \n",
       "4  0.003788  0.000000    0.000000   0.000000     0.000000   0.000000   \n",
       "5  0.003137  0.000000    0.000000   0.000000     0.000000   0.000000   \n",
       "\n",
       "   00 before how  00 buying  00 buying gifts  00 charlie  ...  zoo they sit  \\\n",
       "0       0.000000   0.000000         0.000000    0.000000  ...      0.000000   \n",
       "1       0.000000   0.000000         0.000000    0.002462  ...      0.000000   \n",
       "2       0.000876   0.000876         0.000876    0.000000  ...      0.000876   \n",
       "3       0.000000   0.000000         0.000000    0.000000  ...      0.000000   \n",
       "4       0.000000   0.000000         0.000000    0.000000  ...      0.000000   \n",
       "5       0.000000   0.000000         0.000000    0.000000  ...      0.000000   \n",
       "\n",
       "   zoo three  zoo three times  zoo tickets  zoo tickets cost   zoo zoe  \\\n",
       "0   0.000000         0.000000     0.000000          0.000000  0.000000   \n",
       "1   0.000000         0.000000     0.000000          0.000000  0.002462   \n",
       "2   0.000876         0.000876     0.000000          0.000000  0.000000   \n",
       "3   0.000000         0.000000     0.002568          0.002568  0.000000   \n",
       "4   0.000000         0.000000     0.000000          0.000000  0.000000   \n",
       "5   0.000000         0.000000     0.000000          0.000000  0.000000   \n",
       "\n",
       "   zoo zoe took  zookeeper  zookeeper has  zookeeper has 68  \n",
       "0      0.000000   0.000000       0.000000          0.000000  \n",
       "1      0.002462   0.000000       0.000000          0.000000  \n",
       "2      0.000000   0.000876       0.000876          0.000876  \n",
       "3      0.000000   0.000000       0.000000          0.000000  \n",
       "4      0.000000   0.000000       0.000000          0.000000  \n",
       "5      0.000000   0.000000       0.000000          0.000000  \n",
       "\n",
       "[6 rows x 88944 columns]"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_names = tfidf_vectorizer.get_feature_names_out()\n",
    "tfidf_df = pd.DataFrame(tfidf_vector.toarray(), columns=feature_names)\n",
    "tfidf_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "65cd1a4a-0487-429b-af1b-98b77e9fe1ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "idf_df = pd.DataFrame({'word': tfidf_df.columns,\n",
    "             'idf': tfidf_vectorizer.idf_})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "8375e019-2ec4-403a-abd2-285b38441e33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>idf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00</td>\n",
       "      <td>1.336472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00 and</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00 and all</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00 and he</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00 and they</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88939</th>\n",
       "      <td>zoo zoe</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88940</th>\n",
       "      <td>zoo zoe took</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88941</th>\n",
       "      <td>zookeeper</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88942</th>\n",
       "      <td>zookeeper has</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88943</th>\n",
       "      <td>zookeeper has 68</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>88944 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   word       idf\n",
       "0                    00  1.336472\n",
       "1                00 and  2.252763\n",
       "2            00 and all  2.252763\n",
       "3             00 and he  2.252763\n",
       "4           00 and they  2.252763\n",
       "...                 ...       ...\n",
       "88939           zoo zoe  2.252763\n",
       "88940      zoo zoe took  2.252763\n",
       "88941         zookeeper  2.252763\n",
       "88942     zookeeper has  2.252763\n",
       "88943  zookeeper has 68  2.252763\n",
       "\n",
       "[88944 rows x 2 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idf_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "c969d3b0-696a-4636-95dc-9a7883afd1dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>level_0</th>\n",
       "      <th>level_1</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>00 and</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>00 and all</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>00 and he</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>00 and they</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>533659</th>\n",
       "      <td>5</td>\n",
       "      <td>zoo zoe</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>533660</th>\n",
       "      <td>5</td>\n",
       "      <td>zoo zoe took</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>533661</th>\n",
       "      <td>5</td>\n",
       "      <td>zookeeper</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>533662</th>\n",
       "      <td>5</td>\n",
       "      <td>zookeeper has</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>533663</th>\n",
       "      <td>5</td>\n",
       "      <td>zookeeper has 68</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>533664 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        level_0           level_1    0\n",
       "0             0                00  0.0\n",
       "1             0            00 and  0.0\n",
       "2             0        00 and all  0.0\n",
       "3             0         00 and he  0.0\n",
       "4             0       00 and they  0.0\n",
       "...         ...               ...  ...\n",
       "533659        5           zoo zoe  0.0\n",
       "533660        5      zoo zoe took  0.0\n",
       "533661        5         zookeeper  0.0\n",
       "533662        5     zookeeper has  0.0\n",
       "533663        5  zookeeper has 68  0.0\n",
       "\n",
       "[533664 rows x 3 columns]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_df = tfidf_df.stack().reset_index()\n",
    "tfidf_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "69c9f5d6-2ef9-45b2-87ba-3883f7204f7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>document</th>\n",
       "      <th>term</th>\n",
       "      <th>tfidf</th>\n",
       "      <th>word</th>\n",
       "      <th>idf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>0</td>\n",
       "      <td>marin</td>\n",
       "      <td>0.025597</td>\n",
       "      <td>marin</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>0</td>\n",
       "      <td>and the left</td>\n",
       "      <td>0.021331</td>\n",
       "      <td>and the left</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>0</td>\n",
       "      <td>basket now</td>\n",
       "      <td>0.021331</td>\n",
       "      <td>basket now</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>0</td>\n",
       "      <td>peaches are</td>\n",
       "      <td>0.021331</td>\n",
       "      <td>peaches are</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>0</td>\n",
       "      <td>the basket how</td>\n",
       "      <td>0.021331</td>\n",
       "      <td>the basket how</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>0</td>\n",
       "      <td>the basket now</td>\n",
       "      <td>0.021331</td>\n",
       "      <td>the basket now</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>0</td>\n",
       "      <td>allan</td>\n",
       "      <td>0.017065</td>\n",
       "      <td>allan</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166</th>\n",
       "      <td>0</td>\n",
       "      <td>audrey</td>\n",
       "      <td>0.017065</td>\n",
       "      <td>audrey</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>0</td>\n",
       "      <td>bowls of</td>\n",
       "      <td>0.017065</td>\n",
       "      <td>bowls of</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168</th>\n",
       "      <td>0</td>\n",
       "      <td>four left</td>\n",
       "      <td>0.017065</td>\n",
       "      <td>four left</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88997</th>\n",
       "      <td>1</td>\n",
       "      <td>had ninety</td>\n",
       "      <td>0.034474</td>\n",
       "      <td>had ninety</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89041</th>\n",
       "      <td>1</td>\n",
       "      <td>had seventy</td>\n",
       "      <td>0.022162</td>\n",
       "      <td>had seventy</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89054</th>\n",
       "      <td>1</td>\n",
       "      <td>sheridan</td>\n",
       "      <td>0.019699</td>\n",
       "      <td>sheridan</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89075</th>\n",
       "      <td>1</td>\n",
       "      <td>mrs sheridan</td>\n",
       "      <td>0.017237</td>\n",
       "      <td>mrs sheridan</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89118</th>\n",
       "      <td>1</td>\n",
       "      <td>hair is</td>\n",
       "      <td>0.014774</td>\n",
       "      <td>hair is</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89119</th>\n",
       "      <td>1</td>\n",
       "      <td>of shells</td>\n",
       "      <td>0.014774</td>\n",
       "      <td>of shells</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89120</th>\n",
       "      <td>1</td>\n",
       "      <td>pounds of shells</td>\n",
       "      <td>0.014774</td>\n",
       "      <td>pounds of shells</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89121</th>\n",
       "      <td>1</td>\n",
       "      <td>red roses</td>\n",
       "      <td>0.014774</td>\n",
       "      <td>red roses</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89181</th>\n",
       "      <td>1</td>\n",
       "      <td>15 of them</td>\n",
       "      <td>0.012312</td>\n",
       "      <td>15 of them</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89182</th>\n",
       "      <td>1</td>\n",
       "      <td>capes</td>\n",
       "      <td>0.012312</td>\n",
       "      <td>capes</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177922</th>\n",
       "      <td>2</td>\n",
       "      <td>able</td>\n",
       "      <td>0.042030</td>\n",
       "      <td>able</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177923</th>\n",
       "      <td>2</td>\n",
       "      <td>able to</td>\n",
       "      <td>0.042030</td>\n",
       "      <td>able to</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177978</th>\n",
       "      <td>2</td>\n",
       "      <td>species</td>\n",
       "      <td>0.020140</td>\n",
       "      <td>species</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177990</th>\n",
       "      <td>2</td>\n",
       "      <td>was able</td>\n",
       "      <td>0.018388</td>\n",
       "      <td>was able</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177991</th>\n",
       "      <td>2</td>\n",
       "      <td>was able to</td>\n",
       "      <td>0.018388</td>\n",
       "      <td>was able to</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178032</th>\n",
       "      <td>2</td>\n",
       "      <td>stones</td>\n",
       "      <td>0.014010</td>\n",
       "      <td>stones</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178051</th>\n",
       "      <td>2</td>\n",
       "      <td>lbs</td>\n",
       "      <td>0.013135</td>\n",
       "      <td>lbs</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178052</th>\n",
       "      <td>2</td>\n",
       "      <td>robots</td>\n",
       "      <td>0.013135</td>\n",
       "      <td>robots</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178067</th>\n",
       "      <td>2</td>\n",
       "      <td>forest</td>\n",
       "      <td>0.012259</td>\n",
       "      <td>forest</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178068</th>\n",
       "      <td>2</td>\n",
       "      <td>have in total</td>\n",
       "      <td>0.012259</td>\n",
       "      <td>have in total</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266874</th>\n",
       "      <td>3</td>\n",
       "      <td>min</td>\n",
       "      <td>0.041093</td>\n",
       "      <td>min</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266923</th>\n",
       "      <td>3</td>\n",
       "      <td>about how</td>\n",
       "      <td>0.025683</td>\n",
       "      <td>about how</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266941</th>\n",
       "      <td>3</td>\n",
       "      <td>about how many</td>\n",
       "      <td>0.023115</td>\n",
       "      <td>about how many</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266942</th>\n",
       "      <td>3</td>\n",
       "      <td>was feet</td>\n",
       "      <td>0.023115</td>\n",
       "      <td>was feet</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266965</th>\n",
       "      <td>3</td>\n",
       "      <td>eight hundred</td>\n",
       "      <td>0.020546</td>\n",
       "      <td>eight hundred</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266991</th>\n",
       "      <td>3</td>\n",
       "      <td>how many full</td>\n",
       "      <td>0.017978</td>\n",
       "      <td>how many full</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266992</th>\n",
       "      <td>3</td>\n",
       "      <td>many full</td>\n",
       "      <td>0.017978</td>\n",
       "      <td>many full</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266993</th>\n",
       "      <td>3</td>\n",
       "      <td>uniforms</td>\n",
       "      <td>0.017978</td>\n",
       "      <td>uniforms</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267021</th>\n",
       "      <td>3</td>\n",
       "      <td>hundred sixty</td>\n",
       "      <td>0.015410</td>\n",
       "      <td>hundred sixty</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267022</th>\n",
       "      <td>3</td>\n",
       "      <td>machine can</td>\n",
       "      <td>0.015410</td>\n",
       "      <td>machine can</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355786</th>\n",
       "      <td>4</td>\n",
       "      <td>continues</td>\n",
       "      <td>0.114936</td>\n",
       "      <td>continues</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355787</th>\n",
       "      <td>4</td>\n",
       "      <td>continues how</td>\n",
       "      <td>0.114936</td>\n",
       "      <td>continues how</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355788</th>\n",
       "      <td>4</td>\n",
       "      <td>continues how many</td>\n",
       "      <td>0.114936</td>\n",
       "      <td>continues how many</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355789</th>\n",
       "      <td>4</td>\n",
       "      <td>if this</td>\n",
       "      <td>0.114936</td>\n",
       "      <td>if this</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355790</th>\n",
       "      <td>4</td>\n",
       "      <td>if this pattern</td>\n",
       "      <td>0.114936</td>\n",
       "      <td>if this pattern</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355791</th>\n",
       "      <td>4</td>\n",
       "      <td>pattern</td>\n",
       "      <td>0.114936</td>\n",
       "      <td>pattern</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355792</th>\n",
       "      <td>4</td>\n",
       "      <td>pattern continues</td>\n",
       "      <td>0.114936</td>\n",
       "      <td>pattern continues</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355793</th>\n",
       "      <td>4</td>\n",
       "      <td>pattern continues how</td>\n",
       "      <td>0.114936</td>\n",
       "      <td>pattern continues how</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355794</th>\n",
       "      <td>4</td>\n",
       "      <td>this pattern</td>\n",
       "      <td>0.114936</td>\n",
       "      <td>this pattern</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355795</th>\n",
       "      <td>4</td>\n",
       "      <td>this pattern continues</td>\n",
       "      <td>0.114936</td>\n",
       "      <td>this pattern continues</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444750</th>\n",
       "      <td>5</td>\n",
       "      <td>age</td>\n",
       "      <td>0.054988</td>\n",
       "      <td>age</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444768</th>\n",
       "      <td>5</td>\n",
       "      <td>left over what</td>\n",
       "      <td>0.038068</td>\n",
       "      <td>left over what</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444769</th>\n",
       "      <td>5</td>\n",
       "      <td>over what</td>\n",
       "      <td>0.038068</td>\n",
       "      <td>over what</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444770</th>\n",
       "      <td>5</td>\n",
       "      <td>over what is</td>\n",
       "      <td>0.038068</td>\n",
       "      <td>over what is</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444772</th>\n",
       "      <td>5</td>\n",
       "      <td>km</td>\n",
       "      <td>0.034896</td>\n",
       "      <td>km</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444776</th>\n",
       "      <td>5</td>\n",
       "      <td>smallest</td>\n",
       "      <td>0.032781</td>\n",
       "      <td>smallest</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444778</th>\n",
       "      <td>5</td>\n",
       "      <td>is the smallest</td>\n",
       "      <td>0.031724</td>\n",
       "      <td>is the smallest</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444779</th>\n",
       "      <td>5</td>\n",
       "      <td>the smallest</td>\n",
       "      <td>0.031724</td>\n",
       "      <td>the smallest</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444784</th>\n",
       "      <td>5</td>\n",
       "      <td>age of</td>\n",
       "      <td>0.028551</td>\n",
       "      <td>age of</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444787</th>\n",
       "      <td>5</td>\n",
       "      <td>ages of</td>\n",
       "      <td>0.027494</td>\n",
       "      <td>ages of</td>\n",
       "      <td>2.252763</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        document                    term     tfidf                    word  \\\n",
       "90             0                   marin  0.025597                   marin   \n",
       "119            0            and the left  0.021331            and the left   \n",
       "120            0              basket now  0.021331              basket now   \n",
       "121            0             peaches are  0.021331             peaches are   \n",
       "122            0          the basket how  0.021331          the basket how   \n",
       "123            0          the basket now  0.021331          the basket now   \n",
       "165            0                   allan  0.017065                   allan   \n",
       "166            0                  audrey  0.017065                  audrey   \n",
       "167            0                bowls of  0.017065                bowls of   \n",
       "168            0               four left  0.017065               four left   \n",
       "88997          1              had ninety  0.034474              had ninety   \n",
       "89041          1             had seventy  0.022162             had seventy   \n",
       "89054          1                sheridan  0.019699                sheridan   \n",
       "89075          1            mrs sheridan  0.017237            mrs sheridan   \n",
       "89118          1                 hair is  0.014774                 hair is   \n",
       "89119          1               of shells  0.014774               of shells   \n",
       "89120          1        pounds of shells  0.014774        pounds of shells   \n",
       "89121          1               red roses  0.014774               red roses   \n",
       "89181          1              15 of them  0.012312              15 of them   \n",
       "89182          1                   capes  0.012312                   capes   \n",
       "177922         2                    able  0.042030                    able   \n",
       "177923         2                 able to  0.042030                 able to   \n",
       "177978         2                 species  0.020140                 species   \n",
       "177990         2                was able  0.018388                was able   \n",
       "177991         2             was able to  0.018388             was able to   \n",
       "178032         2                  stones  0.014010                  stones   \n",
       "178051         2                     lbs  0.013135                     lbs   \n",
       "178052         2                  robots  0.013135                  robots   \n",
       "178067         2                  forest  0.012259                  forest   \n",
       "178068         2           have in total  0.012259           have in total   \n",
       "266874         3                     min  0.041093                     min   \n",
       "266923         3               about how  0.025683               about how   \n",
       "266941         3          about how many  0.023115          about how many   \n",
       "266942         3                was feet  0.023115                was feet   \n",
       "266965         3           eight hundred  0.020546           eight hundred   \n",
       "266991         3           how many full  0.017978           how many full   \n",
       "266992         3               many full  0.017978               many full   \n",
       "266993         3                uniforms  0.017978                uniforms   \n",
       "267021         3           hundred sixty  0.015410           hundred sixty   \n",
       "267022         3             machine can  0.015410             machine can   \n",
       "355786         4               continues  0.114936               continues   \n",
       "355787         4           continues how  0.114936           continues how   \n",
       "355788         4      continues how many  0.114936      continues how many   \n",
       "355789         4                 if this  0.114936                 if this   \n",
       "355790         4         if this pattern  0.114936         if this pattern   \n",
       "355791         4                 pattern  0.114936                 pattern   \n",
       "355792         4       pattern continues  0.114936       pattern continues   \n",
       "355793         4   pattern continues how  0.114936   pattern continues how   \n",
       "355794         4            this pattern  0.114936            this pattern   \n",
       "355795         4  this pattern continues  0.114936  this pattern continues   \n",
       "444750         5                     age  0.054988                     age   \n",
       "444768         5          left over what  0.038068          left over what   \n",
       "444769         5               over what  0.038068               over what   \n",
       "444770         5            over what is  0.038068            over what is   \n",
       "444772         5                      km  0.034896                      km   \n",
       "444776         5                smallest  0.032781                smallest   \n",
       "444778         5         is the smallest  0.031724         is the smallest   \n",
       "444779         5            the smallest  0.031724            the smallest   \n",
       "444784         5                  age of  0.028551                  age of   \n",
       "444787         5                 ages of  0.027494                 ages of   \n",
       "\n",
       "             idf  \n",
       "90      2.252763  \n",
       "119     2.252763  \n",
       "120     2.252763  \n",
       "121     2.252763  \n",
       "122     2.252763  \n",
       "123     2.252763  \n",
       "165     2.252763  \n",
       "166     2.252763  \n",
       "167     2.252763  \n",
       "168     2.252763  \n",
       "88997   2.252763  \n",
       "89041   2.252763  \n",
       "89054   2.252763  \n",
       "89075   2.252763  \n",
       "89118   2.252763  \n",
       "89119   2.252763  \n",
       "89120   2.252763  \n",
       "89121   2.252763  \n",
       "89181   2.252763  \n",
       "89182   2.252763  \n",
       "177922  2.252763  \n",
       "177923  2.252763  \n",
       "177978  2.252763  \n",
       "177990  2.252763  \n",
       "177991  2.252763  \n",
       "178032  2.252763  \n",
       "178051  2.252763  \n",
       "178052  2.252763  \n",
       "178067  2.252763  \n",
       "178068  2.252763  \n",
       "266874  2.252763  \n",
       "266923  2.252763  \n",
       "266941  2.252763  \n",
       "266942  2.252763  \n",
       "266965  2.252763  \n",
       "266991  2.252763  \n",
       "266992  2.252763  \n",
       "266993  2.252763  \n",
       "267021  2.252763  \n",
       "267022  2.252763  \n",
       "355786  2.252763  \n",
       "355787  2.252763  \n",
       "355788  2.252763  \n",
       "355789  2.252763  \n",
       "355790  2.252763  \n",
       "355791  2.252763  \n",
       "355792  2.252763  \n",
       "355793  2.252763  \n",
       "355794  2.252763  \n",
       "355795  2.252763  \n",
       "444750  2.252763  \n",
       "444768  2.252763  \n",
       "444769  2.252763  \n",
       "444770  2.252763  \n",
       "444772  2.252763  \n",
       "444776  2.252763  \n",
       "444778  2.252763  \n",
       "444779  2.252763  \n",
       "444784  2.252763  \n",
       "444787  2.252763  "
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_df = tfidf_df.rename(columns={0:'tfidf', 'level_0': 'document','level_1': 'term', 'level_2': 'term'})\n",
    "view_tfidf = tfidf_df.sort_values(by=['document','tfidf'], ascending=[True,False])\n",
    "view_tfidf = pd.merge(view_tfidf, idf_df, how='left', left_on='term', right_on='word')\n",
    "view_tfidf=view_tfidf[view_tfidf['idf']==2.252762968495368]\n",
    "view_tfidf.groupby('document').head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "531ef3fa-249b-454a-86da-63278ae99240",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.252762968495368"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idf_df['idf'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52279b66-edb5-4b3f-a5d0-d2c60f76d275",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}