GetIdentifierFromIA.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:6ba6a50f7b059509eb7a38420917b1c256326ba0a309187b6a2cc6e103f5848b"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import requests\n",
      "import json\n",
      "import csv\n",
      "import codecs\n",
      "import cStringIO"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#UnicodeWriter from http://docs.python.org/2/library/csv.html#examples\n",
      "class UnicodeWriter:\n",
      "    def __init__(self, f, dialect=csv.excel, encoding=\"utf-8-sig\", **kwds):\n",
      "        self.queue = cStringIO.StringIO()\n",
      "        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)\n",
      "        self.stream = f\n",
      "        self.encoder = codecs.getincrementalencoder(encoding)()\n",
      "    def writerow(self, row):\n",
      "        self.writer.writerow([s.encode(\"utf-8\") for s in row])\n",
      "        data = self.queue.getvalue()\n",
      "        data = data.decode(\"utf-8\")\n",
      "        data = self.encoder.encode(data)\n",
      "        self.stream.write(data)\n",
      "        self.queue.truncate(0)\n",
      "\n",
      "    def writerows(self, rows):\n",
      "        for row in rows:\n",
      "            self.writerow(row)\n",
      "\n",
      "#/end UnicodeWriter"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#goal is to take all the json results from Internet Archive API and print into a CSV file\n",
      "\n",
      "resultFile = open(\"IAresults.csv\",\"wb\")\n",
      "\n",
      "url = \"https://archive.org/advancedsearch.php?q=%28astro*%29+AND+mediatype%3Atexts+AND+date%3A[1800-01-01+TO+1950-12-31]+NOT+%28arXiv%29&fl[]=call_number&fl[]=collection&fl[]=contributor&fl[]=creator&fl[]=date&fl[]=format&fl[]=identifier&fl[]=publisher&fl[]=rights&fl[]=source&fl[]=subject&fl[]=title&fl[]=type&fl[]=volume&fl[]=week&fl[]=year&sort[]=&sort[]=&sort[]=&rows=10000&page=1&output=json&save=yes#raw\"\n",
      "content = requests.get(url)\n",
      "k=content.json() \n",
      "wr = UnicodeWriter(resultFile,dialect='excel',quoting=csv.QUOTE_ALL)\n",
      "\n",
      "#identify the number of results\n",
      "num = k['response']['numFound']\n",
      "print num\n",
      "#loop through the results printing to csv\n",
      "for i in range(0, num):\n",
      "    ID = k['response']['docs'][i]['identifier']\n",
      "    print ID\n",
      "    \n",
      "    try:\n",
      "        title = k['response']['docs'][i]['title']\n",
      "    except KeyError:\n",
      "        title = \"No Title\"\n",
      "    print title\n",
      "    \n",
      "    try:\n",
      "        creator = k['response']['docs'][i]['creator']\n",
      "        creators = \"; \".join(creator)\n",
      "    except KeyError:\n",
      "        creators = \"No Creator\"\n",
      "    print creators\n",
      "    \n",
      "    try:\n",
      "        publisher = k['response']['docs'][i]['publisher']\n",
      "        publishers = \"; \".join(publisher)\n",
      "    except KeyError:\n",
      "        publishers = \"No Publisher\"\n",
      "    print publishers\n",
      "    \n",
      "    try:\n",
      "        contributor = k['response']['docs'][i]['contributor']\n",
      "        contributors = \"; \".join(contributor)\n",
      "    except KeyError:\n",
      "        contributors = \"No Contributor\"\n",
      "    print contributors\n",
      "    \n",
      "    try:\n",
      "        dates = k['response']['docs'][i]['date']\n",
      "        #dates = \"; \".join(date)\n",
      "    except KeyError:\n",
      "        dates = \"No Date\"\n",
      "    print dates\n",
      "    \n",
      "    \n",
      "    wr.writerow([ID]+[title]+[creators]+[publishers]+[contributors]+[dates])\n",
      "\n",
      "resultFile.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "ConnectionError",
       "evalue": "('Connection aborted.', error(10060, 'A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond'))",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mConnectionError\u001b[0m                           Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-3-3834e19cb41e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0murl\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"https://archive.org/advancedsearch.php?q=%28astro*%29+AND+mediatype%3Atexts+AND+date%3A[1800-01-01+TO+1950-12-31]+NOT+%28arXiv%29&fl[]=call_number&fl[]=collection&fl[]=contributor&fl[]=creator&fl[]=date&fl[]=format&fl[]=identifier&fl[]=publisher&fl[]=rights&fl[]=source&fl[]=subject&fl[]=title&fl[]=type&fl[]=volume&fl[]=week&fl[]=year&sort[]=&sort[]=&sort[]=&rows=10000&page=1&output=json&save=yes#raw\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m \u001b[0mcontent\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcontent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[0mwr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mUnicodeWriter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresultFile\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdialect\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'excel'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mquoting\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcsv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mQUOTE_ALL\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32mC:\\Users\\Megan\\Anaconda\\lib\\site-packages\\requests\\api.pyc\u001b[0m in \u001b[0;36mget\u001b[1;34m(url, **kwargs)\u001b[0m\n\u001b[0;32m     57\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     58\u001b[0m     \u001b[0mkwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msetdefault\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'allow_redirects'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 59\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mrequest\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'get'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     60\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     61\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32mC:\\Users\\Megan\\Anaconda\\lib\\site-packages\\requests\\api.pyc\u001b[0m in \u001b[0;36mrequest\u001b[1;34m(method, url, **kwargs)\u001b[0m\n\u001b[0;32m     46\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     47\u001b[0m     \u001b[0msession\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msessions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 48\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0murl\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     49\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     50\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32mC:\\Users\\Megan\\Anaconda\\lib\\site-packages\\requests\\sessions.pyc\u001b[0m in \u001b[0;36mrequest\u001b[1;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert)\u001b[0m\n\u001b[0;32m    449\u001b[0m         }\n\u001b[0;32m    450\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 451\u001b[1;33m         \u001b[0mresp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    452\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    453\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mresp\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32mC:\\Users\\Megan\\Anaconda\\lib\\site-packages\\requests\\sessions.pyc\u001b[0m in \u001b[0;36msend\u001b[1;34m(self, request, **kwargs)\u001b[0m\n\u001b[0;32m    555\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    556\u001b[0m         \u001b[1;31m# Send the request\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 557\u001b[1;33m         \u001b[0mr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0madapter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    558\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    559\u001b[0m         \u001b[1;31m# Total elapsed time of the request (approximately)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32mC:\\Users\\Megan\\Anaconda\\lib\\site-packages\\requests\\adapters.pyc\u001b[0m in \u001b[0;36msend\u001b[1;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[0;32m    405\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    406\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mProtocolError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msocket\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 407\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mConnectionError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mrequest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    408\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    409\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mMaxRetryError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mConnectionError\u001b[0m: ('Connection aborted.', error(10060, 'A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond'))"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}