index.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Digital Library of India Kannada Deduplication\n",
    "\n",
    "\n",
    "Author: Arjuna Rao Chavala,(arjunaraoc@gmail.com)\n",
    "\n",
    "Date of inital draft: 2018-11-28\n",
    "\n",
    "\n",
    "\n",
    "This study explores the work done to identify duplicate items of Kannada collection of  Digital Library of India.  Original file size of the item is identified as the key parameter to identify duplicates. The study is presented as Jupyter notebook with code in Python 3 so that the research becomes reproducible\n",
    "\n",
    "\n",
    "For more detail on the methodology and problems with DlI, please read the [Deduplication study done for Telugu Collection](https://github.com/arjunaraoc/Deduplicate-DLI).\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the metadata from archive.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#some preliminaries Python modules used by the code in the notebook\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import csv\n",
    "from IPython.display import Image\n",
    "import requests\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#code for downloading catalog. commented as the live collection could change the output file is provided\n",
    "#gets archive item fields and DLI description subfields\n",
    "def getCollection2(resfile,numitems):\n",
    "    try:\n",
    "        fo=open(resfile,\"w\")\n",
    "        error_log = open('arxerrlog.txt', 'w+')\n",
    "        url = \"https://archive.org/services/search/v1/scrape?\"\n",
    "        basic_params={ 'q':'(collection%3Adigitallibraryindia+AND+(language%3Akan++OR+language%3AKannada))',\n",
    "                       'fields':'identifier,title,creator,date,description'}\n",
    "        params=basic_params.copy()\n",
    "        numline = 0\n",
    "        fo.write( \"id\"+\"\\t\"+\"title\"+\"\\t\"+\"creator\"+\"\\t\"+\"pubd\"+\"\\t\"+\"pages\"+\"\\t\"+\"bc\"+\"\\n\")\n",
    "        while True:\n",
    "            try:\n",
    "\n",
    "                params_str= \"&\".join(\"%s=%s\" % (k, v) for k, v in params.items())\n",
    "                print (params_str)\n",
    "                resp = requests.get(url+params_str, headers={})\n",
    "            except requests.exceptions.RequestException as e:  # This is the correct syntax\n",
    "                error_log.write('Could not get search result' + url + params+' because of error: %s\\n' % e)\n",
    "                print (\"There was an error; writing to log.\")\n",
    "                sys.exit(1)\n",
    "            else:\n",
    "                data= resp.json()\n",
    "                #write results\n",
    "                iadict=data[\"items\"]\n",
    "                for i in iadict:\n",
    "                    iaid=i['identifier']\n",
    "\n",
    "                    iatitle=\"\"\n",
    "                    if 'title' in i:\n",
    "                        iatitle=i['title']\n",
    "                    iacreator=\"\"\n",
    "                    if 'creator' in i:\n",
    "                        iacreator= i['creator']\n",
    "                    iadate=\"\"\n",
    "                    if 'date' in i:\n",
    "                        iadate= i['date']\n",
    "                    iadesc=\"\"\n",
    "                    iadesc_totpages=\"\"\n",
    "                    iadesc_barcode=\"\"\n",
    "                    if 'description' in i:\n",
    "                        iadesc=i['description']\n",
    "                        \n",
    "                        totpagessearchstr = \"dc.description.totalpages\" + \": \" + \"([0-9]+)\"\n",
    "                        m = re.search(totpagessearchstr,iadesc)\n",
    "                        if m:\n",
    "                            iadesc_totpages = m.group(1)\n",
    "                        \n",
    "                        # barcode search\n",
    "                        bcsearchstr = \"dc.identifier.barcode\" + \": \" + \"([0-9]+)\"\n",
    "                        m = re.search(bcsearchstr,iadesc)\n",
    "                        if m:\n",
    "                            iadesc_barcode = m.group(1)\n",
    "\n",
    "\n",
    "                    fo.writelines(\"%s\\t%s\\t%s\\t%s\\t%s\\t%s\\n\" % (iaid,iatitle,iacreator,iadate,\n",
    "                                                                    iadesc_totpages,iadesc_barcode))\n",
    "                    numline += 1\n",
    "                    if (numitems != 0) and (numline > numitems):\n",
    "                        break\n",
    "                if (numitems != 0) and (numline > numitems):\n",
    "                    break\n",
    "                cursor = data.get('cursor', None)\n",
    "                print(cursor)\n",
    "                if cursor is None:\n",
    "                    break\n",
    "                else:\n",
    "                    params = basic_params.copy()\n",
    "                    params['cursor'] = cursor\n",
    "        fo.close()\n",
    "    except IOError:\n",
    "        print (\"Error: can\\'t find file or read data\")\n",
    "#getCollection2(\"./data/arxkancat.tsv\",0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dlicat=pd.read_csv(\"./data/arxkancat.tsv\",index_col=None, sep=\"\\t\",converters={i: str for i in range(1,100)})\n",
    "dlicat.to_csv(\"./data/arxkanfin.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final dataset\n",
    "\n",
    "The data is organised as identifier(archive.org, title, creator, publication date(pubd), pages and DLI barcode(bc)\n",
    "Barcode contains information as follows( Number of digits given after the field )\n",
    "<center no:1 or 2>,<Vendor number:2><Scanning location:3><source library:3><item number:7>\n",
    "\n",
    "\n",
    "If you look at the samples(below), even bc was not captured properly by DLI as evidenced by all 9s for some of the fields."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>creator</th>\n",
       "      <th>pubd</th>\n",
       "      <th>pages</th>\n",
       "      <th>bc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dli.osmania.3040</td>\n",
       "      <td>ಮುಂದಿನ ದೇವರು</td>\n",
       "      <td>ಕೆ.ಕೆ.ಶೆಟ್ಟಿ</td>\n",
       "      <td>1936-01-01T00:00:00Z</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dli.osmania.3041</td>\n",
       "      <td>ಕುರುಡು ಓದು</td>\n",
       "      <td>ಮಾನಪ್ಪ</td>\n",
       "      <td>1946-01-01T00:00:00Z</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dli.osmania.3042</td>\n",
       "      <td>ಎಚ್ಚತ್ತ ಆಗ್ನೇಯ ಏಶಿಯ</td>\n",
       "      <td>ಎಂ. ಹರಿದಾಸ</td>\n",
       "      <td>1942-01-01T00:00:00Z</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dli.osmania.3043</td>\n",
       "      <td>ಕರುಣಾಲಹರಿ</td>\n",
       "      <td>ಎಂ.ವಿ. ಸೀತಾರಾಮಯ್ಯ</td>\n",
       "      <td>1955-01-01T00:00:00Z</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dli.osmania.3044</td>\n",
       "      <td>ಪುಟ್ಟರಸು</td>\n",
       "      <td>ಹೊಯಿಸಳ</td>\n",
       "      <td>1949-01-01T00:00:00Z</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                title            creator  \\\n",
       "0  dli.osmania.3040         ಮುಂದಿನ ದೇವರು       ಕೆ.ಕೆ.ಶೆಟ್ಟಿ   \n",
       "1  dli.osmania.3041           ಕುರುಡು ಓದು             ಮಾನಪ್ಪ   \n",
       "2  dli.osmania.3042  ಎಚ್ಚತ್ತ ಆಗ್ನೇಯ ಏಶಿಯ         ಎಂ. ಹರಿದಾಸ   \n",
       "3  dli.osmania.3043            ಕರುಣಾಲಹರಿ  ಎಂ.ವಿ. ಸೀತಾರಾಮಯ್ಯ   \n",
       "4  dli.osmania.3044             ಪುಟ್ಟರಸು             ಹೊಯಿಸಳ   \n",
       "\n",
       "                   pubd pages bc  \n",
       "0  1936-01-01T00:00:00Z           \n",
       "1  1946-01-01T00:00:00Z           \n",
       "2  1942-01-01T00:00:00Z           \n",
       "3  1955-01-01T00:00:00Z           \n",
       "4  1949-01-01T00:00:00Z           "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat=pd.read_csv(\"./data/arxkanfin.csv\",index_col=0,converters={i: str for i in range(1,100)})\n",
    "cat.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>creator</th>\n",
       "      <th>pubd</th>\n",
       "      <th>pages</th>\n",
       "      <th>bc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5339</th>\n",
       "      <td>in.ernet.dli.2015.494839</td>\n",
       "      <td>೧೯೮೬_ಜುಲೈ_ಸಪ್ತಗಿರಿ__ಕನ್ನಡ</td>\n",
       "      <td>ಕೆ. ಸುಬ್ಬರಾವ್</td>\n",
       "      <td>1986-01-01T00:00:00Z</td>\n",
       "      <td>44</td>\n",
       "      <td>02040100074126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5340</th>\n",
       "      <td>in.ernet.dli.2015.494840</td>\n",
       "      <td>ಸಪ್ತಗಿರಿ ಜೂನ್ ೧೯೮೬ ಕನ್ನಡ</td>\n",
       "      <td>ಕೆ. ಸುಬ್ಬರಾವ್</td>\n",
       "      <td>1986-01-01T00:00:00Z</td>\n",
       "      <td>44</td>\n",
       "      <td>02040100074127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5341</th>\n",
       "      <td>in.ernet.dli.2015.494841</td>\n",
       "      <td>೧೯೮೬ ಮಾರ್ಚ್ ಸಪ್ತಗಿರಿ ಕನ್ನಡ</td>\n",
       "      <td>ಕೆ. ಸುಬ್ಬರಾವ್</td>\n",
       "      <td>1986-01-01T00:00:00Z</td>\n",
       "      <td>42</td>\n",
       "      <td>02040100074128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5342</th>\n",
       "      <td>in.ernet.dli.2015.494842</td>\n",
       "      <td>ಸಪ್ತಗಿರಿ ಕನ್ನಡ ಮೇ ೧೯೮೬</td>\n",
       "      <td>ಕೆ. ಸುಬ್ಬರಾವ್</td>\n",
       "      <td>1986-01-01T00:00:00Z</td>\n",
       "      <td>43</td>\n",
       "      <td>02040100074129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5343</th>\n",
       "      <td>in.ernet.dli.2015.494843</td>\n",
       "      <td>೧೯೮೬ ನವೆಂಬರ್ ಸಪ್ತಗಿರಿ ಕನ್ನಡ</td>\n",
       "      <td>ಕೆ. ಸುಬ್ಬರಾವ್</td>\n",
       "      <td>1986-01-01T00:00:00Z</td>\n",
       "      <td>44</td>\n",
       "      <td>02040100074130</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            id                        title        creator  \\\n",
       "5339  in.ernet.dli.2015.494839    ೧೯೮೬_ಜುಲೈ_ಸಪ್ತಗಿರಿ__ಕನ್ನಡ  ಕೆ. ಸುಬ್ಬರಾವ್   \n",
       "5340  in.ernet.dli.2015.494840     ಸಪ್ತಗಿರಿ ಜೂನ್ ೧೯೮೬ ಕನ್ನಡ  ಕೆ. ಸುಬ್ಬರಾವ್   \n",
       "5341  in.ernet.dli.2015.494841   ೧೯೮೬ ಮಾರ್ಚ್ ಸಪ್ತಗಿರಿ ಕನ್ನಡ  ಕೆ. ಸುಬ್ಬರಾವ್   \n",
       "5342  in.ernet.dli.2015.494842       ಸಪ್ತಗಿರಿ ಕನ್ನಡ ಮೇ ೧೯೮೬  ಕೆ. ಸುಬ್ಬರಾವ್   \n",
       "5343  in.ernet.dli.2015.494843  ೧೯೮೬ ನವೆಂಬರ್ ಸಪ್ತಗಿರಿ ಕನ್ನಡ  ಕೆ. ಸುಬ್ಬರಾವ್   \n",
       "\n",
       "                      pubd pages              bc  \n",
       "5339  1986-01-01T00:00:00Z    44  02040100074126  \n",
       "5340  1986-01-01T00:00:00Z    44  02040100074127  \n",
       "5341  1986-01-01T00:00:00Z    42  02040100074128  \n",
       "5342  1986-01-01T00:00:00Z    43  02040100074129  \n",
       "5343  1986-01-01T00:00:00Z    44  02040100074130  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "in.ernet.dli    3125\n",
       "dli.osmania     2219\n",
       "Name: 0, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Type of digital library project\n",
    "cat['id'].str.extract(r'(in\\.ernet\\.dli|dli\\.osmania)')[0].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "       2219\n",
       "001     478\n",
       "002    2105\n",
       "010     441\n",
       "999     101\n",
       "Name: bc, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Scanning location code and number of items, first row had blank denoting osmania library items\n",
    "#blank-Osmania, 001-IIIT, Allahabad ,002-Osmania ,010-SVDL,999-CDAC \n",
    "cat['bc'].str[-10:-7].value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Summary info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>creator</th>\n",
       "      <th>pubd</th>\n",
       "      <th>pages</th>\n",
       "      <th>bc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5344</td>\n",
       "      <td>5344</td>\n",
       "      <td>5344</td>\n",
       "      <td>5344</td>\n",
       "      <td>5344</td>\n",
       "      <td>5344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>5344</td>\n",
       "      <td>4299</td>\n",
       "      <td>2565</td>\n",
       "      <td>274</td>\n",
       "      <td>616</td>\n",
       "      <td>3126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>in.ernet.dli.2015.363050</td>\n",
       "      <td>ಬಸವರಾಜದೇವರ ರಗಳೆ</td>\n",
       "      <td>ಕೆ. ಸುಬ್ಬರಾವ್</td>\n",
       "      <td>1955-01-01T00:00:00Z</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>145</td>\n",
       "      <td>376</td>\n",
       "      <td>2219</td>\n",
       "      <td>2219</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              id            title        creator  \\\n",
       "count                       5344             5344           5344   \n",
       "unique                      5344             4299           2565   \n",
       "top     in.ernet.dli.2015.363050  ಬಸವರಾಜದೇವರ ರಗಳೆ  ಕೆ. ಸುಬ್ಬರಾವ್   \n",
       "freq                           1                4            145   \n",
       "\n",
       "                        pubd pages    bc  \n",
       "count                   5344  5344  5344  \n",
       "unique                   274   616  3126  \n",
       "top     1955-01-01T00:00:00Z              \n",
       "freq                     376  2219  2219  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat.describe(include='all')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " Uniqueness based on various parameters is given below. (output cell shows the count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5081"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfg1=cat.groupby(['title','creator'])\n",
    "dfg1.ngroups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5120"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfg2=cat.groupby(['title','creator','pubd'])\n",
    "dfg2.ngroups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5307"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfg3=cat.groupby(['title','creator','pubd','pages'])\n",
    "dfg3.ngroups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "csvfile = open('data/flagdupset.csv', 'w', newline=\"\")\n",
    "writer = csv.writer(csvfile, delimiter=\",\")\n",
    "for name, group in dfg3:\n",
    "    if len(group)>1:\n",
    "        duplist=group['id'].tolist()\n",
    "        writer.writerow(duplist)\n",
    "csvfile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['in.ernet.dli.2015.287352,in.ernet.dli.2015.447771\\n',\n",
       " 'in.ernet.dli.2015.287354,in.ernet.dli.2015.447773\\n',\n",
       " 'in.ernet.dli.2015.381941,in.ernet.dli.2015.382264\\n',\n",
       " 'in.ernet.dli.2015.382196,in.ernet.dli.2015.382198\\n',\n",
       " 'dli.osmania.3245,dli.osmania.516\\n']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('data/flagdupset.csv') as fi:\n",
    "    dup=fi.readlines()\n",
    "dup[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# of duplicate lines\n",
    "len(dup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "103"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# count of all ids in by counting \",\" and adding number of lines\n",
    "numdupids=[len(s.split(\",\")) for s in dup]\n",
    "sum(numdupids)+len(dup)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Size access"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# takes long time if there are more items. As the collection could change,output file provided.\n",
    "# for duplicates csv file, get size, output duplicates,sizes,comparison status using api call for speedup\n",
    "def sizeCompareForDuplicates2(inpfile,outpfile, numlines):\n",
    "    import subprocess\n",
    "    import json\n",
    "    url = \"https://archive.org/metadata/\"\n",
    "    try:\n",
    "        fo=open(outpfile,\"w\")\n",
    "        error_log = open('arxerrlog.txt', 'w+')\n",
    "        line=1\n",
    "        result = []\n",
    "        resultset=set()\n",
    "        fi=open(inpfile,\"r\")\n",
    "        for row in fi.readlines():\n",
    "            row=row.strip(\"\\n\")\n",
    "            idlist=row.split(sep=\",\")\n",
    "            index=0\n",
    "            result.clear()\n",
    "            resultset.clear()\n",
    "            for id in idlist:\n",
    "                params_str = \"%s/files\" % id\n",
    "                print(params_str)\n",
    "                try:\n",
    "                    resp = requests.get(url + params_str, headers={})\n",
    "                except requests.exceptions.RequestException as e:  # This is the correct syntax\n",
    "                    error_log.write('Could not get search result' + url + params + ' because of error: %s\\n' % e)\n",
    "                    print(\"There was an error; writing to log.\")\n",
    "                    sys.exit(1)\n",
    "                else:\n",
    "                    data = resp.json()['result']\n",
    "\n",
    "                size='0'\n",
    "                for obj in data:\n",
    "                    if obj['name'].find(\".pdf\")!= -1:\n",
    "                        size=obj['size']\n",
    "                        break\n",
    "                if(int(size)==0):\n",
    "                    print(\"Error, Did not find pdf file for determining size\")\n",
    "                    exit(-1)\n",
    "                result.append(size)\n",
    "                index+=1\n",
    "            #compare resulting sizes\n",
    "            resultset=set(result)\n",
    "            if len(resultset)==1:\n",
    "                 compare=\"Success\"\n",
    "            else:\n",
    "                compare=\"Fail\"\n",
    "            #write resultline\n",
    "            index=0;\n",
    "            for id in idlist:\n",
    "                fo.write(id+\",\"+result[index]+\",\")\n",
    "                index+=1\n",
    "            fo.write(compare+\"\\n\")\n",
    "            print(line,compare)\n",
    "            line += 1\n",
    "            if (numlines != 0) and (line > numlines):\n",
    "                break\n",
    "\n",
    "    except IOError:\n",
    "        print(\"Error: can\\'t find file or read data\")\n",
    "#sizeCompareForDuplicates2(\"./data/flagdupset.csv\",\"./data/flagdupsetcompare.csv\",0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['in.ernet.dli.2015.287352,7577942,in.ernet.dli.2015.447771,7577942,Success\\n',\n",
       " 'in.ernet.dli.2015.287354,9381409,in.ernet.dli.2015.447773,9381409,Success\\n',\n",
       " 'in.ernet.dli.2015.381941,6461914,in.ernet.dli.2015.382264,6461914,Success\\n',\n",
       " 'in.ernet.dli.2015.382196,89377000,in.ernet.dli.2015.382198,89377000,Success\\n',\n",
       " 'dli.osmania.3245,14195488,dli.osmania.516,16042444,Fail\\n']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('data/flagdupsetcompare.csv') as fi:\n",
    "    dupcompare=fi.readlines()\n",
    "dupcompare[0:5]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum('Success' in s for s in dupcompare)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum('Fail'  in s for s in dupcompare )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read from the duplicates size comparison output, when there is success, write the ids, when there is a fail,\n",
    "# find subsets which have samesize, write their ids, and also write uniques if exist() using csv module\n",
    "def splitdup_size(inpfile, outfile,numlines):\n",
    "    import pandas as pd\n",
    "    import numpy as np\n",
    "    import csv\n",
    "\n",
    "    line = 0\n",
    "    fi = open(inpfile, 'r')\n",
    "    csvfile = open(outfile, 'w', newline=\"\")\n",
    "    writer = csv.writer(csvfile, delimiter=\",\")\n",
    "    for row in fi.readlines():\n",
    "        line += 1\n",
    "        if row.find(\"Fail\") != -1:\n",
    "            row = row.replace(\",Fail\\n\", \"\")\n",
    "            info = row.split(\",\")\n",
    "            ids = []\n",
    "            sizes = []\n",
    "            for i, j in zip(info[0::2], info[1::2]):\n",
    "                ids.append(i)\n",
    "                sizes.append(j)\n",
    "            isf = pd.DataFrame({'id': pd.Series(ids), 'size': sizes})\n",
    "            isfg = isf.groupby(['size'], sort=False)\n",
    "\n",
    "            for name, group in isfg:\n",
    "                duplist = group['id'].tolist()\n",
    "                writer.writerow(duplist)\n",
    "        else:\n",
    "            row = row.replace(\",Success\\n\", \"\")\n",
    "            info = row.split(\",\")\n",
    "            ids = []\n",
    "            sizes = []\n",
    "            for i, j in zip(info[0::2], info[1::2]):\n",
    "                ids.append(i)\n",
    "                sizes.append(j)\n",
    "            isf = pd.DataFrame({'id': pd.Series(ids), 'size': sizes})\n",
    "            isfg = isf.groupby(['size'], sort=False)\n",
    "            for name, group in isfg:\n",
    "                duplist = group['id'].tolist()\n",
    "                writer.writerow(duplist)\n",
    "        if (numlines != 0) and (line > numlines):\n",
    "            break\n",
    "\n",
    "    csvfile.close()\n",
    "\n",
    "splitdup_size('data/flagdupsetcompare.csv','data/flagdupsetrevised.csv',0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "48"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fi=open(\"./data/flagdupsetrevised.csv\",'r')\n",
    "lines=[]\n",
    "for row in fi.readlines():\n",
    "    lines.append(row)\n",
    "\n",
    "len(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['in.ernet.dli.2015.287352,in.ernet.dli.2015.447771\\n',\n",
       " 'in.ernet.dli.2015.287354,in.ernet.dli.2015.447773\\n',\n",
       " 'in.ernet.dli.2015.381941,in.ernet.dli.2015.382264\\n',\n",
       " 'in.ernet.dli.2015.382196,in.ernet.dli.2015.382198\\n',\n",
       " 'dli.osmania.3245\\n']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#show sample \n",
    "lines[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Non duplicate lines  based on size,only id will be present on line, without comma\n",
    "nondupl=[i for i in lines if i.find(\",\")==-1]\n",
    "len(nondupl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#duplicate lines  based on size,comma will be present, this number also ids which are to be flagged.\n",
    "dupl=[i for i in lines if i.find(\",\")!=-1]\n",
    "len(dupl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# of ids to be deleted\n",
    "numids=[len(s.split(\",\")) for s in dupl]\n",
    "numdups=sum(numids)-len(dupl)\n",
    "numdups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4116766467065868"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of duplicates as percentage\n",
    "numdups*100/cat.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Shell Script generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "ffo=open(\"./data/dup_flag.sh\",\"w\")\n",
    "fdo=open(\"./data/dup_delete.sh\",\"w\")\n",
    "fi=open(\"./data/flagdupsetrevised.csv\",'r')\n",
    "line=0\n",
    "import time\n",
    "for row in fi.readlines():\n",
    "    line+=1\n",
    "    row=row.strip(\"\\n\")\n",
    "    ids=row.split(\",\")\n",
    "    for i in range(0,len(ids)):\n",
    "        if i==0:\n",
    "            if len(ids)>1:\n",
    "                #write notes command\n",
    "                curation_notes=row.replace(ids[0]+\",\",\"\")\n",
    "                notescommand=(\"ia metadata %s  --modify=\\\"notes:Exact duplicates (Archive identifiers) of \" \n",
    "                              \"this item, which are likely to be \"\n",
    "                              \"hidden or deleted: %s\\\"\\n\") % (ids[0], curation_notes)  \n",
    "                ffo.write(notescommand)\n",
    "            \n",
    "        else:\n",
    "            fdo.write(\"ia delete \"+ids[i]+\" -H x-archive-keep-old-version:0\\n\")\n",
    "    \n",
    "\n",
    "ffo.close()\n",
    "fdo.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}