0-usgs-api.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download 'horizontal' + 'vertical' Landsat ARD tiles via Earth Explorer API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import getpass\n",
    "import os\n",
    "import tarfile\n",
    "from urllib.parse import urlparse\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_apikey(username, password):\n",
    "    payload = dict(username=username, password=password, catalogId='EE', authType='EROS')\n",
    "    data = dict(jsonRequest=json.dumps(payload))\n",
    "    r = requests.post('https://earthexplorer.usgs.gov/inventory/json/v/1.4.0/login', data=data)\n",
    "    creds = r.json()\n",
    "    apiKey = creds['data']\n",
    "    return apiKey"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " ··········\n"
     ]
    }
   ],
   "source": [
    "username = 'scottyhq'\n",
    "password = getpass.getpass()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "apiKey = get_apikey(username, password)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_scenes(region, horizontal, vertical):\n",
    "    myfilter = {\n",
    "            \"filterType\": \"and\", \n",
    "            \"childFilters\": [\n",
    "                {\"filterType\":\"value\",\"fieldId\":21789,\"value\":region}, #Grid Region\n",
    "                {\"filterType\":\"value\",\"fieldId\":21787,\"value\":horizontal}, #Horizontal\n",
    "                {\"filterType\":\"value\",\"fieldId\":21788,\"value\":vertical}, #Vertical\n",
    "            ]\n",
    "        }\n",
    "\n",
    "    request_code = 'search'\n",
    "    baseurl = f'https://earthexplorer.usgs.gov/inventory/json/v/1.4.0/{request_code}'\n",
    "    payload = {'apiKey':apiKey, \n",
    "               'datasetName': 'ARD_TILE', \n",
    "               'additionalCriteria': myfilter,  \n",
    "               'maxResults':10000}\n",
    "    data = dict(jsonRequest=json.dumps(payload))\n",
    "    r = requests.get(baseurl, params=data)\n",
    "    response = r.json()\n",
    "    return response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Query USGS Earth Explorer API for scenes\n",
    "region='AK'\n",
    "horizontal='008'\n",
    "vertical='003'\n",
    "response = get_scenes(region, horizontal, vertical)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2595 scenes\n",
      "Returned 2595 scenes\n",
      "First acquisition:  1984-06-18\n",
      "Last acquisition:  2019-03-01\n"
     ]
    }
   ],
   "source": [
    "print('Found {} scenes'.format(response['data']['totalHits']))\n",
    "print('Returned {} scenes'.format(response['data']['numberReturned']))\n",
    "print('First acquisition: ', response['data']['results'][0]['acquisitionDate'])\n",
    "print('Last acquisition: ', response['data']['results'][-1]['acquisitionDate'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_url(entityId, apiKey, product='ST'):\n",
    "    request_code = 'download'\n",
    "    baseurl = f'https://earthexplorer.usgs.gov/inventory/json/v/1.4.0/{request_code}'\n",
    "    payload = {'apiKey':apiKey, \n",
    "               'datasetName': 'ARD_TILE', \n",
    "               'entityIds':[entityId],\n",
    "               'products':[product]} #these are 'download codes' from above\n",
    "    data = dict(jsonRequest=json.dumps(payload))\n",
    "    r = requests.get(baseurl, params=data)\n",
    "    download_url = r.json()['data'][0]['url']\n",
    "    p = urlparse(download_url)\n",
    "    name = os.path.basename(p.path)\n",
    "    return name, download_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def download(filename, url):\n",
    "    chunks = 1024*1024\n",
    "    with requests.get(url, stream=True) as r:\n",
    "        with open(filename, 'wb') as f:\n",
    "            for chunk in r.iter_content(chunk_size=chunks):\n",
    "                f.write(chunk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# grab a file for testing\n",
    "ids = [x['entityId'] for x in response['data']['results']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LT05_AK_008003_19840618_C01_V01\n",
      "LT05_AK_008003_19840711_C01_V01\n",
      "no downloadable product for that id...\n",
      "LT05_AK_008003_19840720_C01_V01\n",
      "no downloadable product for that id...\n",
      "LT05_AK_008003_19840729_C01_V01\n",
      "no downloadable product for that id...\n",
      "LT05_AK_008003_19840812_C01_V01\n"
     ]
    }
   ],
   "source": [
    "# Download all in serial (try first 5)\n",
    "success = []\n",
    "fail = []\n",
    "for entityId in ids[:5]:\n",
    "    print(entityId)\n",
    "    try:\n",
    "        tarname, url = get_url(entityId, apiKey)\n",
    "        download(tarname, url)\n",
    "        success.append(entityId)\n",
    "    except IndexError:\n",
    "        print('no downloadable product for that id...')\n",
    "        fail.append(entityId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}