aws_processing/a3_image_gener (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "difficult-silence",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import re\n",
    "import zipfile\n",
    "import getpass\n",
    "from osgeo import gdal \n",
    "import os  # for chdir, getcwd, path.basename, path.exists\n",
    "import pandas as pd # for DatetimeIndex\n",
    "import codecs # for text parsing code\n",
    "import netrc\n",
    "import rasterio as rio\n",
    "import glob\n",
    "import io\n",
    "import shutil\n",
    "from subprocess import PIPE, Popen\n",
    "import subprocess\n",
    "import fcntl ##may need to pip install this one\n",
    "import select\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "under-liechtenstein",
   "metadata": {},
   "outputs": [],
   "source": [
    "def downloading(file, amp=False):\n",
    "    \"\"\"\n",
    "    Downloads and unzips UAVSAR images from ASF Vertex. Only tested on .GRD Interferometric Pairs. \n",
    "    Ideally for this application only pass 1 url at a time.\n",
    "    :param zip_url: url pointing at a UAVSAR .zip file\n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    # Get NASA EARTHDATA Credentials from ~/.netrc or manual input\n",
    "    try:\n",
    "        os.chmod('/home/jovyan/.netrc', 0o600) #only necessary on jupyterhub\n",
    "        (ASF_USER, account, ASF_PASS) = netrc.netrc().authenticators(\"urs.earthdata.nasa.gov\")\n",
    "    except:\n",
    "        ASF_USER = input(\"Enter Username: \")\n",
    "        ASF_PASS = getpass.getpass(\"Enter Password: \")\n",
    "        \n",
    "        \n",
    "    data_dir = '/tmp/'\n",
    "   \n",
    "    # directory for data downloads\n",
    "\n",
    "    os.makedirs(data_dir, exist_ok=True)\n",
    "    os.chdir(data_dir)\n",
    "\n",
    "    print(f'downloading {file}...')\n",
    "    filename = os.path.basename(file)\n",
    "https://datapool.asf.alaska.edu/AMPLITUDE_GRD/UA/uticam_21003_21004-002_21013-003_0034d_s01_L090_01_amp_grd.zip\n",
    "    if not os.path.exists(os.path.join(data_dir,filename)):\n",
    "        \n",
    "        if amp:\n",
    "            amp_file = file.replace('int','amp')\n",
    "            \n",
    "        \n",
    "        cmd = \"wget -1 {0} --user={1} --password={2} -P {3} --progress=bar:force\".format(file, ASF_USER, ASF_PASS, data_dir)\n",
    "        #os.system(cmd) \n",
    "        #subprocess.call(cmd)\n",
    "        process = Popen(['wget',file,'--user={}'.format(ASF_USER),'--password={}'.format(ASF_PASS),'-P',data_dir,'--progress=bar'], stderr=subprocess.PIPE)\n",
    "        started = False\n",
    "        for line in process.stderr:\n",
    "            line = line.decode(\"utf-8\", \"replace\")\n",
    "            if started:\n",
    "                splited = line.split()\n",
    "                if len(splited) == 9:\n",
    "                    percentage = splited[6]\n",
    "                    speed = splited[7]\n",
    "                    remaining = splited[8]\n",
    "                    print(\"Downloaded {} with {} per second and {} left.\".format(percentage, speed, remaining), end='\\r')\n",
    "            elif line == os.linesep:\n",
    "                started = True\n",
    "\n",
    "        ##Should probably be a subprocess.call(cmd) - not quite sure why but that is the perfered method\n",
    "    else:\n",
    "        print(filename + \" already exists. Skipping download ..\")\n",
    "\n",
    "    #print(\"\\nDone\")\n",
    "    \n",
    "    # unzip\n",
    "\n",
    "    for file in glob.glob(\"/tmp/*.zip\"):\n",
    "        with zipfile.ZipFile(file, \"r\") as zip_ref:\n",
    "            print('Extracting all the files now...')\n",
    "            zip_ref.extractall('/tmp')\n",
    "            print(\"done\")\n",
    "    \n",
    "    return data_dir\n",
    "\n",
    "# folder is path to a folder with an .ann (or .txt) and .grd files (.amp1, .amp2, .cor, .unw, .int)\n",
    "\n",
    "def uavsar_tiff_convert(folder, verbose = False):\n",
    "    \"\"\"\n",
    "    Builds a header file for the input UAVSAR .grd file,\n",
    "    allowing the data to be read as a raster dataset.\n",
    "    :param folder:   the folder containing the UAVSAR .grd and .ann files\n",
    "    \"\"\"\n",
    "\n",
    "    os.chdir(folder)\n",
    "    int_file = glob.glob(os.path.join(folder, 'int.grd'))\n",
    "\n",
    "    # Empty lists to put information that will be recalled later.\n",
    "    Lines_list = []\n",
    "    Samples_list = []\n",
    "    Latitude_list = []\n",
    "    Longitude_list = []\n",
    "    Files_list = []\n",
    "\n",
    "    # Step 1: Look through folder and determine how many different flights there are\n",
    "    # by looking at the HDR files.\n",
    "    for files in os.listdir(folder):\n",
    "        if files [-4:] == \".grd\":\n",
    "            newfile = open(files[0:-4] + \".hdr\", 'w')\n",
    "            newfile.write(\"\"\"ENVI\n",
    "description = {DESCFIELD}\n",
    "samples = NSAMP\n",
    "lines = NLINE\n",
    "bands = 1\n",
    "header offset = 0\n",
    "data type = DATTYPE\n",
    "interleave = bsq\n",
    "sensor type = UAVSAR L-Band\n",
    "byte order = 0\n",
    "map info = {Geographic Lat/Lon, \n",
    "            1.000, \n",
    "            1.000, \n",
    "            LON, \n",
    "            LAT,  \n",
    "            0.0000555600000000, \n",
    "            0.0000555600000000, \n",
    "            WGS-84, units=Degrees}\n",
    "wavelength units = Unknown\n",
    "                \"\"\"\n",
    "                          )\n",
    "            newfile.close()\n",
    "            if files[0:18] not in Files_list:\n",
    "                Files_list.append(files[0:18])\n",
    "\n",
    "    #Variables used to recall indexed values.\n",
    "    var1 = 0\n",
    "\n",
    "    #Step 2: Look through the folder and locate the annotation file(s).\n",
    "    # These can be in either .txt or .ann file types.\n",
    "    for files in os.listdir(folder):\n",
    "        if Files_list[var1] and files[-4:] == \".txt\" or files[-4:] == \".ann\":\n",
    "            #Step 3: Once located, find the info we are interested in and append it to\n",
    "            # the appropriate list. We limit the variables to <=1 so that they only\n",
    "            # return two values (one for each polarization of\n",
    "            searchfile = codecs.open(files, encoding = 'windows-1252', errors='ignore')\n",
    "            for line in searchfile:\n",
    "                if \"Ground Range Data Latitude Lines\" in line:\n",
    "                    Lines = line[65:70]\n",
    "                    if verbose:\n",
    "                        print(f\"Number of Lines: {Lines}\")\n",
    "                    if Lines not in Lines_list:\n",
    "                        Lines_list.append(Lines)\n",
    "\n",
    "                elif \"Ground Range Data Longitude Samples\" in line:\n",
    "                    Samples = line[65:70]\n",
    "                    if verbose:\n",
    "                        print(f\"Number of Samples: {Samples}\")\n",
    "                    if Samples not in Samples_list:\n",
    "                        Samples_list.append(Samples)\n",
    "\n",
    "                elif \"Ground Range Data Starting Latitude\" in line:\n",
    "                    Latitude = line[65:85]\n",
    "                    if verbose:\n",
    "                        print(f\"Top left lat: {Latitude}\")\n",
    "                    if Latitude not in Latitude_list:\n",
    "                        Latitude_list.append(Latitude)\n",
    "\n",
    "                elif \"Ground Range Data Starting Longitude\" in line:\n",
    "                    Longitude = line[65:85]\n",
    "                    if verbose:\n",
    "                        print(f\"Top left Lon: {Longitude}\")\n",
    "                    if Longitude not in Longitude_list:\n",
    "                        Longitude_list.append(Longitude)\n",
    "    \n",
    "                        \n",
    "                 \n",
    "            #Reset the variables to zero for each different flight date.\n",
    "            var1 = 0\n",
    "            searchfile.close()\n",
    "\n",
    "\n",
    "    # Step 3: Open .hdr file and replace data for all type 4 (real numbers) data\n",
    "    # this all the .grd files expect for .int\n",
    "    for files in os.listdir(folder):\n",
    "        if files[-4:] == \".hdr\":\n",
    "            with open(files, \"r\") as sources:\n",
    "                lines = sources.readlines()\n",
    "            with open(files, \"w\") as sources:\n",
    "                for line in lines:\n",
    "                    if \"data type = DATTYPE\" in line:\n",
    "                        sources.write(re.sub(line[12:19], \"4\", line))\n",
    "                    elif \"DESCFIELD\" in line:\n",
    "                        sources.write(re.sub(line[15:24], folder, line))\n",
    "                    elif \"lines\" in line:\n",
    "                        sources.write(re.sub(line[8:13], Lines, line))\n",
    "                    elif \"samples\" in line:\n",
    "                        sources.write(re.sub(line[10:15], Samples, line))\n",
    "                    elif \"LAT\" in line:\n",
    "                        sources.write(re.sub(line[12:15], Latitude, line))\n",
    "                    elif \"LON\" in line:\n",
    "                        sources.write(re.sub(line[12:15], Longitude, line))\n",
    "                    else:\n",
    "                        sources.write(re.sub(line, line, line))\n",
    "    \n",
    "    # Step 3: Open .hdr file and replace data for .int file date type 6 (complex)                 \n",
    "    for files in os.listdir(folder):\n",
    "        if files[-8:] == \".int.hdr\":\n",
    "            with open(files, \"r\") as sources:\n",
    "                lines = sources.readlines()\n",
    "            with open(files, \"w\") as sources:\n",
    "                for line in lines:\n",
    "                    if \"data type = 4\" in line:\n",
    "                        sources.write(re.sub(line[12:13], \"6\", line))\n",
    "                    elif \"DESCFIELD\" in line:\n",
    "                        sources.write(re.sub(line[15:24], folder, line))\n",
    "                    elif \"lines\" in line:\n",
    "                        sources.write(re.sub(line[8:13], Lines, line))\n",
    "                    elif \"samples\" in line:\n",
    "                        sources.write(re.sub(line[10:15], Samples, line))\n",
    "                    elif \"LAT\" in line:\n",
    "                        sources.write(re.sub(line[12:15], Latitude, line))\n",
    "                    elif \"LON\" in line:\n",
    "                        sources.write(re.sub(line[12:15], Longitude, line))\n",
    "                    else:\n",
    "                        sources.write(re.sub(line, line, line))\n",
    "                        \n",
    "    \n",
    "    # Step 4: Now we have an .hdr file, the data is geocoded and can be loaded into python with rasterio\n",
    "    # once loaded in we use gdal.Translate to convert and save as a .tiff\n",
    "    \n",
    "    data_to_process = glob.glob(os.path.join(folder, '*.grd')) # list all .grd files\n",
    "    for data_path in data_to_process: # loop to open and translate .grd to .tiff, and save .tiffs using gdal\n",
    "        raster_dataset = gdal.Open(data_path, gdal.GA_ReadOnly)\n",
    "        raster = gdal.Translate(os.path.join(folder, os.path.basename(data_path) + '.tiff'), raster_dataset, format = 'Gtiff', outputType = gdal.GDT_Float32)\n",
    "    \n",
    "    # Step 5: Save the .int raster, needs separate save because of the complex format\n",
    "    data_to_process = glob.glob(os.path.join(folder, '*.int.grd')) # list all .int.grd files (only 1)\n",
    "    for data_path in data_to_process:\n",
    "        raster_dataset = gdal.Open(data_path, gdal.GA_ReadOnly)\n",
    "        raster = gdal.Translate(os.path.join(folder, os.path.basename(data_path) + '.tiff'), raster_dataset, format = 'Gtiff', outputType = gdal.GDT_CFloat32)\n",
    "\n",
    "    print(\".tiffs have been created\")\n",
    "    return\n",
    "\n",
    "def a3_bucket_transfer(folder):\n",
    "    \"\"\"\n",
    "    transfers converted .tiff files to the a3 cloud\n",
    "    :param folder:  (filepath) to folder containing the UAVSAR .tiff\n",
    "    :param region: (string) the region the flight is from\n",
    "    \"\"\"\n",
    "    num_tiffs = len(glob.glob(folder+ \"*.tiff\"))\n",
    "    \n",
    "    for tiff in glob.glob(folder+ \"*.tiff\"):\n",
    "        base_name = tiff.split('/')[-1]\n",
    "        \n",
    "        region = base_name.split('_')[0]\n",
    "        year = '20' + base_name.split('_')[2][0:2]\n",
    "        flight_num = base_name.split('_')[2][2:5]\n",
    "        flight_heading = base_name.split('_')[1][0:3]\n",
    "        days_between = base_name.split('_')[4]\n",
    "        folder_name = '{}_{}_{}_{}_{}/'.format(region,year,flight_num, flight_heading, days_between)\n",
    "        \n",
    "        tiff_folder_fp = os.path.join(folder, folder_name)\n",
    "        if not os.path.exists(tiff_folder_fp):\n",
    "            os.mkdir(tiff_folder_fp)\n",
    "        \n",
    "\n",
    "        os.replace(tiff,tiff_folder_fp+base_name )\n",
    "    cmd = 'aws s3 cp {} s3://snowex-data/uavsar-project/UAVSAR_images/{} --recursive'.format(tiff_folder_fp,folder_name)\n",
    "    os.system(cmd)\n",
    "    \n",
    "    ###now check for upload complete (probably a smoother way to do this. This is has a danger of hanging)\n",
    "    upload_incomplete = True\n",
    "    if upload_incomplete:\n",
    "        cmd = 'aws s3 ls s3://snowex-data/uavsar-project/UAVSAR_images/{} | wc -l'.format(folder_name)\n",
    "        stream = os.popen(cmd)\n",
    "        output = stream.read()\n",
    "        if int(output) == num_tiffs:\n",
    "            upload_incomplete = False\n",
    "            print('upload complete')\n",
    "    \n",
    "    return folder_name, output\n",
    "\n",
    "def clear_folder(folder):\n",
    "    if os.path.exists(folder):\n",
    "        shutil.rmtree(folder, ignore_errors = True)\n",
    "    return\n",
    "\n",
    "def main(zip_url, clear_temp = True):\n",
    "    zip_num = len(zip_url)\n",
    "    count = 1\n",
    "    for url in zip_url:\n",
    "        print('Starting {} of {} zips'.format(count, zip_num))\n",
    "        count += 1\n",
    "        data_dir = downloading(url)\n",
    "        uavsar_tiff_convert(data_dir)\n",
    "        folder_name, number_uploaded = a3_bucket_transfer(data_dir)\n",
    "        if clear_temp:\n",
    "            clear_folder(data_dir)\n",
    "        print('Created folder {} with {} images'.format(folder_name, number_uploaded.strip()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "alternate-brazil",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "test_file = 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/silver_34715_20011-001_20016-002_0019d_s01_L090_01_int_grd.zip'\n",
    "downloading(test_file)\n",
    "\n",
    "folder = '/tmp/'\n",
    "uavsar_tiff_convert(folder)\n",
    "\n",
    "folder = '/tmp/'\n",
    "a3_bucket_transfer(folder)\n",
    "\n",
    "folder = '/tmp/'\n",
    "clear_folder(folder)\n",
    "\n",
    "test_files = ['https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_21019-017_21021-005_0006d_s01_L090_01_int_grd.zip']\n",
    "main(test_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "swedish-smith",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_21019-017_21021-005_0006d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_21017-017_21019-017_0006d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_21016-002_21017-017_0007d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_21010-002_21011-010_0007d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_21007-004_21010-002_0007d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_20003-028_20017-006_0040d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_20013-004_20017-006_0015d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_20003-028_20013-004_0025d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_20008-004_20013-004_0007d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_20003-028_20008-004_0018d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_20005-007_20008-004_0007d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_20003-028_20005-007_0011d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/GrMesa_26108_16095-005_17091-005_0313d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/GrMesa_08112_16095-004_17091-004_0313d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_08006_17016-002_17043-006_0037d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17018-001_17043-005_0034d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17002-001_17043-005_0053d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17016-001_17043-005_0037d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_08006_17002-002_17043-006_0053d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_07805_17018-000_17043-004_0034d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_07805_17016-000_17043-004_0037d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_07805_17002-000_17043-004_0053d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_07805_17002-000_17026-005_0030d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_07805_17016-000_17026-005_0014d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_08006_17002-002_17026-002_0030d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_08006_17016-002_17026-002_0014d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17002-001_17026-001_0030d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17016-001_17026-001_0014d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17018-001_17026-001_0011d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17016-001_17018-001_0003d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17002-001_17018-001_0019d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17016-001_17018-001_0003d_s01_L090_02_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_07805_17016-000_17018-000_0003d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_07805_17002-000_17018-000_0019d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_08006_17002-002_17016-002_0016d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17002-001_17016-001_0016d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17002-001_17016-001_0016d_s01_L090_02_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_07805_17002-000_17016-000_0016d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_07805_17002-000_17016-000_0016d_s01_L090_02_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/GrMesa_26108_15060-006_15100-005_0048d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/GrMesa_08112_15060-005_15100-004_0048d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/GrMesa_26108_15055-007_15060-006_0008d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/GrMesa_08112_15055-006_15060-005_0008d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/GrMesa_26108_15054-005_15055-007_0006d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/GrMesa_08112_15054-004_15055-006_0006d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/GrMesa_08112_15100-004_16058-000_0360d_s01_L090_01_int_grd.zip', 'https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_21011-010_21016-002_0021d_s01_L090_01_int_grd.zip']\n"
     ]
    }
   ],
   "source": [
    "grand_mesa_txt = open('/home/jovyan/uavsar/aws_processing/grmesa_zip_fps.txt')\n",
    "content = grand_mesa_txt.read()\n",
    "grand_mesa_zips = content.split(\"\\n\")\n",
    "grand_mesa_txt.close()\n",
    "print(grand_mesa_zips)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "talented-refrigerator",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting 1 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_21019-017_21021-005_0006d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 236M per second and 0s left.....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2021_019_274_0006d/ with 16 images\n",
      "Starting 2 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_21017-017_21019-017_0006d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 154M per second and 0s left....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2021_017_274_0006d/ with 16 images\n",
      "Starting 3 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_21016-002_21017-017_0007d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 170M per second and 0s left.....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2021_016_274_0007d/ with 16 images\n",
      "Starting 4 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_21010-002_21011-010_0007d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 1.48M per second and 0s left..t.\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2021_010_274_0007d/ with 16 images\n",
      "Starting 5 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_21007-004_21010-002_0007d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 4.08M per second and 0s left....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2021_007_274_0007d/ with 16 images\n",
      "Starting 6 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_20003-028_20017-006_0040d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 155M per second and 0s left....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2020_003_274_0040d/ with 4 images\n",
      "Starting 7 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_20013-004_20017-006_0015d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 291M per second and 0s left.....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2020_013_274_0015d/ with 16 images\n",
      "Starting 8 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_20003-028_20013-004_0025d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 207M per second and 0s left....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2020_003_274_0025d/ with 4 images\n",
      "Starting 9 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_20008-004_20013-004_0007d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 108M per second and 0s left...t.\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2020_008_274_0007d/ with 16 images\n",
      "Starting 10 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_20003-028_20008-004_0018d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 241M per second and 0s left....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2020_003_274_0018d/ with 4 images\n",
      "Starting 11 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_20005-007_20008-004_0007d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 271M per second and 0s left.....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2020_005_274_0007d/ with 16 images\n",
      "Starting 12 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_27416_20003-028_20005-007_0011d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 190M per second and 0s left.....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2020_003_274_0011d/ with 16 images\n",
      "Starting 13 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/GrMesa_26108_16095-005_17091-005_0313d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 37.3M per second and 0s left..t.\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder GrMesa_2016_095_261_0313d/ with 4 images\n",
      "Starting 14 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/GrMesa_08112_16095-004_17091-004_0313d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 197M per second and 0s left.....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder GrMesa_2016_095_081_0313d/ with 4 images\n",
      "Starting 15 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_08006_17016-002_17043-006_0037d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 202M per second and 0s left....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_016_080_0037d/ with 4 images\n",
      "Starting 16 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17018-001_17043-005_0034d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 309M per second and 0s left....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_018_260_0034d/ with 4 images\n",
      "Starting 17 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17002-001_17043-005_0053d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 16.2M per second and 0s left..t.\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_002_260_0053d/ with 16 images\n",
      "Starting 18 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17016-001_17043-005_0037d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 23.5M per second and 0s left....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_016_260_0037d/ with 16 images\n",
      "Starting 19 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_08006_17002-002_17043-006_0053d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 1.09M per second and 0s left...\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_002_080_0053d/ with 4 images\n",
      "Starting 20 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_07805_17018-000_17043-004_0034d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 93.2M per second and 0s left...\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_018_078_0034d/ with 4 images\n",
      "Starting 21 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_07805_17016-000_17043-004_0037d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 9.53M per second and 0s left...\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_016_078_0037d/ with 4 images\n",
      "Starting 22 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_07805_17002-000_17043-004_0053d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 164M per second and 0s left....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_002_078_0053d/ with 4 images\n",
      "Starting 23 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_07805_17002-000_17026-005_0030d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 268M per second and 0s left....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_002_078_0030d/ with 4 images\n",
      "Starting 24 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_07805_17016-000_17026-005_0014d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 306M per second and 0s left....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_016_078_0014d/ with 4 images\n",
      "Starting 25 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_08006_17002-002_17026-002_0030d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 21.1M per second and 0s left...\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_002_080_0030d/ with 4 images\n",
      "Starting 26 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_08006_17016-002_17026-002_0014d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 58.3M per second and 0s left...\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_016_080_0014d/ with 4 images\n",
      "Starting 27 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17002-001_17026-001_0030d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 278M per second and 0s left.....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_002_260_0030d/ with 16 images\n",
      "Starting 28 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17016-001_17026-001_0014d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 174M per second and 0s left.....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_016_260_0014d/ with 16 images\n",
      "Starting 29 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17018-001_17026-001_0011d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 255M per second and 0s left.....\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_018_260_0011d/ with 16 images\n",
      "Starting 30 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17016-001_17018-001_0003d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 1.24M per second and 0s left...\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n",
      "upload complete\n",
      "Created folder grmesa_2017_016_260_0003d/ with 4 images\n",
      "Starting 31 of 47 zips\n",
      "downloading https://datapool.asf.alaska.edu/INTERFEROMETRY_GRD/UA/grmesa_26006_17002-001_17018-001_0019d_s01_L090_01_int_grd.zip...\n",
      "doneloaded 99% with 95.0M per second and 0s left..t.\n",
      "Extracting all the files now...\n",
      "done\n",
      ".tiffs have been created\n"
     ]
    }
   ],
   "source": [
    "main(grand_mesa_zips)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}