diff --git a/.travis.yml b/.travis.yml index 5bb3dcd..63a97aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,30 +1,33 @@ language: python - python: - - '2.7' - -# Setup anaconda +- '2.7' before_install: - - sudo add-apt-repository -y ppa:ubuntugis/ppa - - sudo apt-get update -qq - - sudo apt-get install libgdal1h gdal-bin libgdal-dev libcurl4-gnutls-dev - - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh - - chmod +x miniconda.sh - - ./miniconda.sh -b - - export PATH=/home/travis/miniconda/bin:$PATH - - conda update --yes conda - # The next couple lines fix a crash with multiprocessing on Travis and are not specific to using Miniconda - - sudo rm -rf /dev/shm - - sudo ln -s /run/shm /dev/shm - -# Install packages +- sudo add-apt-repository -y ppa:ubuntugis/ppa +- sudo apt-get update -qq +- sudo apt-get install libgdal1h gdal-bin libgdal-dev libcurl4-gnutls-dev +- wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh +- chmod +x miniconda.sh +- "./miniconda.sh -b" +- export PATH=/home/travis/miniconda/bin:$PATH +- conda update --yes conda +- sudo rm -rf /dev/shm +- sudo ln -s /run/shm /dev/shm install: - - conda install --yes python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib scikit-image six nose dateutil - - conda install --yes -c https://conda.binstar.org/osgeo gdal - - pip install --install-option="--no-cython-compile" cython - - pip uninstall requests --yes - - pip install requests==2.5.3 - - pip install -r requirements/travis.txt - +- conda install --yes python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib scikit-image + six nose dateutil +- conda install --yes -c https://conda.binstar.org/osgeo gdal +- pip install --install-option="--no-cython-compile" cython +- pip uninstall requests --yes +- pip install requests==2.5.3 +- pip install -r requirements/travis.txt script: - - nosetests +- nosetests +deploy: + provider: pypi + user: devseed + password: + secure: WtawFW/999XYszmZfj1Qk82l00OSyP2JBVOOGCERrW1gVO7MYtYsgP31HKRSzNTCTHJNVDpdK4WZWY6zPQqC3l2UfWYYsvRn0hCoI8AJxE5VCUEg6Ccpe6fMJuhp1pq6Zy7yrfBSZcOB9aqSHLBJsunD2o3mNlTC8WV8vNK74ck= + on: + tags: true + repo: developmentseed/landsat-util + branch: master diff --git a/CHANGES.txt b/CHANGES.txt index 2f0e387..095a540 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,18 @@ Changes ======= +0.7.0 (2015-05-29) +------------------ +- New documentation +- Deployed to readthedocs +- Automate deployment to pypi +- Adds docker support +- skip unzipping if images already unzipped +- add force-unzip flag +- fix a bug where multiple downloads was not followed by multiple process #81 +- fix a bug where if scenes was downloaded from google instead of aws, process failed #84 +- download band 8 when pansharpen fixes #73 + 0.6.3 (2015-04-29) ------------------ - adjust lower rescaling bound, closes #66 for now diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a6b9827 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,4 @@ +FROM ubuntu:14.10 +RUN apt-get -y update +RUN apt-get install --yes python-pip python-numpy python-scipy libgdal-dev libatlas-base-dev gfortran libfreetype6-dev libglib2.0-dev zlib1g-dev python-pycurl +RUN pip install landsat-util diff --git a/README.rst b/README.rst index 61c238f..1578ac6 100644 --- a/README.rst +++ b/README.rst @@ -4,183 +4,40 @@ Landsat-util .. image:: https://travis-ci.org/developmentseed/landsat-util.svg?branch=v0.5 :target: https://travis-ci.org/developmentseed/landsat-util -.. image:: https://pypip.in/version/landsat-util/badge.svg - :target: https://pypi.python.org/pypi/landsat-util/ - :alt: Latest Version +.. image:: https://badge.fury.io/py/landsat-util.svg + :target: http://badge.fury.io/py/landsat-util -.. image:: https://pypip.in/download/landsat-util/badge.svg +.. image:: https://img.shields.io/pypi/dm/landsat-util.svg :target: https://pypi.python.org/pypi/landsat-util/ :alt: Downloads -.. image:: https://pypip.in/wheel/landsat-util/badge.svg - :target: https://pypi.python.org/pypi/landsat-util/ - :alt: Wheel Status - -.. image:: https://pypip.in/license/landsat-util/badge.svg +.. image:: https://img.shields.io/pypi/l/landsat-util.svg :target: https://pypi.python.org/pypi/landsat-util/ :alt: License -Landsat-util is a command line utility that makes it easy to search, download, and process Landsat imagery. - -This tool uses Development Seed's `API for Landsat Metadata `_. - -This API is accessible here: https://api.developmentseed.org/landsat - -You can also run your own API and connect it to this tool. - -Installation -============ - -**On Mac** - - ``$: pip install landsat-util`` - -**On Ubuntu 14.04** - -Use pip to install landsat-util. If you are not using virtualenv, you might have to run ``pip`` as ``sudo``. - - ``$: sudo apt-get update`` - - ``$: sudo apt-get install python-pip python-numpy python-scipy libgdal-dev libatlas-base-dev gfortran`` - - ``$: pip install landsat-util`` - -**On Other systems** - -Make sure Python setuptools is installed. - - ``$: python setup.py numpy six`` - - ``$: python setup.py install`` - - -**To Upgrade** - - ``$: pip install -U landsat-util`` - -If you have installed previous version of landsat using brew, first run: - - ``$: brew uninstall landsat-util`` - -**To Test** - - ``$: pip install -U requirements/dev.txt`` - - ``$: nosetests`` - -Or - - ``$: python setup.py test`` - -Overview: What can landsat-util do? -==================================== - -Landsat-util has three main functions: - -- **Search** for landsat tiles based on several search parameters. -- **Download** landsat images. -- **Image processing** and pan sharpening on landsat images. - -These three functions have to be performed separately. - -**Help**: Type ``landsat -h`` for detailed usage parameters. - -Step 1: Search -=============== - -Search returns information about all landsat tiles that match your criteria. This includes a link to an unprocessed preview of the tile. The most important result is the tile's *sceneID*, which you will need to download the tile (see step 2 below). - -Search for landsat tiles in a given geographical region, using any of the following: - -- **Paths and rows**: If you know the paths and rows you want to search for. -- **Latidue and Longitude**: If you need the latitude and longitude of the point you want to search for. - -Additionally filter your search using the following parameters: - -- **Start and end dates** for when imagery was taken -- **Maximum percent cloud cover** (default is 20%) - -**Examples of search**: - -Search by path and row: - -``$: landsat search --cloud 4 --start "january 1 2014" --end "january 10 2014" -p 009,045`` - -Search by latitude and longitude: - -``$: landsat search --lat 38.9004204 --lon -77.0237117`` - - -Step 2: Download -================= - -You can download tiles using their unique sceneID, which you get from landsat search. - -Landsat-util will download a zip file that includes all the bands. You have the option of specifying the bands you want to down. In this case, landsat-util only downloads those bands if they are available online. - -**Examples of download**: - -Download images by their custom sceneID, which you get from landsat search: - -``$: landsat download LC80090452014008LGN00`` - -Download only band 4, 3 and 2 for a particular sceneID: -``$: landsat download LC80090452014008LGN00 --bands 432`` - -Download multiple sceneIDs: - -``$: landsat download LC80090452014008LGN00 LC80090452015008LGN00 LC80090452013008LGN00`` - -Step 3: Image processing -========================= - -You can process your downloaded tiles with our custom image processing algorithms. In addition, you can choose to pansharpen your images and specify which bands to process. - -**Examples of image processing**: - -Process images that are already downloaded. Remember, the program accepts both zip files and unzipped folders: - -``$: landsat process path/to/LC80090452014008LGN00.tar.bz`` - -If unzipped: - -``$: landsat process path/to/LC80090452014008LGN00`` - -Specify bands 3, 5 and 1: - -``$: landsat process path/to/LC80090452014008LGN00 --bands 351`` - -Process *and* pansharpen a downloaded image: - -``$: landsat process path/to/LC80090452014008LGN00.tar.bz --pansharpen`` - - -Important Notes -=============== +Landsat-util is a command line utility that makes it easy to search, download, and process Landsat imagery. -- All downloaded and processed images are stored at your home directory in landsat forlder: ``~/landsat`` +Docs ++++++ -- The image thumbnail web address that is included in the results can be used to make sure that clouds are not obscuring the subject of interest. Run the search again if you need to narrow down your result and then start downloading images. Each image is usually more than 700mb and it might takes a very long time if there are too many images to download +For full documentation visit: http://landsat-util.readthedocs.org/ -- Image processing is a very heavy and resource consuming task. Each process takes about 5-10 mins. We recommend that you run the processes in smaller badges. Pansharpening, while increasing image resolution 2x, substantially increases processing time. +To run the documentation locally:: -- Landsat-util requires at least 2GB of Memory (RAM). + $ cd docs + $ make html -Recently Added -+++++++++++++++ +Recently Added Features ++++++++++++++++++++++++ - Add longitude latitude search - Improve console output - Add more color options such as false color, true color, etc. +Change Log ++++++++++ + +See `CHANGES.txt `_. -To Do List -++++++++++ -- Add Sphinx Documentation -- Add capacity for NDVI output -- Add alternative projections (currently only option is default web-mercator; EPSG: 3857) -- Connect search to Google Address API -- Include 16-bit image variant in output -- Add support for color correct looping over multiple compressed inputs (currently just 1) diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index cc5949d..0000000 --- a/Vagrantfile +++ /dev/null @@ -1,37 +0,0 @@ - -Vagrant.configure(2) do |config| - config.vm.box = "hashicorp/precise64" - - config.vm.provider "virtualbox" do |v| - v.memory = 2064 - v.cpus = 2 - end - - config.vm.provision "shell", inline: <<-SHELL - apt-get update - apt-get install python-software-properties -y - add-apt-repository -y ppa:ubuntugis/ppa - apt-get update -qq - apt-get install -y build-essential python-dev python-pip libgdal1h gdal-bin libgdal-dev libcurl4-gnutls-dev - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh - chmod +x miniconda.sh - ./miniconda.sh -b - export PATH=/root/miniconda/bin:$PATH - chmod 740 /root - chown -R vagrant:vagrant /root/miniconda - echo export PATH=/root/miniconda/bin:\$PATH >> /home/vagrant/.profile - conda update --yes conda - # The next couple lines fix a crash with multiprocessing on Travis and are not specific to using Miniconda - sudo rm -rf /dev/shm - sudo ln -s /run/shm /dev/shm - - conda install --yes -c https://conda.binstar.org/osgeo gdal - conda install --yes numpy scikit-image requests nose dateutil - - pip install --install-option="--no-cython-compile" cython - pip install -r /vagrant/requirements/travis.txt - cd /vagrant - nosetests - - SHELL -end diff --git a/docs/Makefile b/docs/Makefile index 4e4ddd5..a0c4ec9 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -19,7 +19,7 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext help: @echo "Please use \`make ' where is one of" @@ -30,6 +30,7 @@ help: @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @@ -45,6 +46,7 @@ help: @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* @@ -85,17 +87,25 @@ qthelp: @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/landsat.qhcp" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Landsat-util.qhcp" @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/landsat.qhc" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Landsat-util.qhc" + +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/landsat" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/landsat" + @echo "# mkdir -p $$HOME/.local/share/devhelp/Landsat-util" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Landsat-util" @echo "# devhelp" epub: @@ -166,6 +176,11 @@ doctest: @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo diff --git a/docs/landsat.rst b/docs/api.rst similarity index 50% rename from docs/landsat.rst rename to docs/api.rst index 730e50f..6ecbb31 100644 --- a/docs/landsat.rst +++ b/docs/api.rst @@ -1,77 +1,58 @@ -landsat package -=============== +Module Index +============ -Subpackages ------------ - -.. toctree:: - - landsat.tests - -Submodules ----------- - -landsat.downloader module -------------------------- +downloader.py ++++++++++++++++++++++++++ .. automodule:: landsat.downloader :members: :undoc-members: :show-inheritance: -landsat.image module --------------------- +uploader.py ++++++++++++++++++++++++++ -.. automodule:: landsat.image +.. automodule:: landsat.uploader :members: :undoc-members: :show-inheritance: -landsat.landsat module ----------------------- +image.py ++++++++++++++++++++++++++ -.. automodule:: landsat.landsat +.. automodule:: landsat.image :members: :undoc-members: :show-inheritance: -landsat.mixins module ---------------------- +landsat.py ++++++++++++++++++++++++++ -.. automodule:: landsat.mixins +.. automodule:: landsat.landsat :members: :undoc-members: :show-inheritance: -landsat.search module ---------------------- +mixins.py +++++++++++++++++++++++ -.. automodule:: landsat.search +.. automodule:: landsat.mixins :members: :undoc-members: :show-inheritance: -landsat.settings module ------------------------ +search.py +++++++++++++++++++++++ -.. automodule:: landsat.settings +.. automodule:: landsat.search :members: :undoc-members: :show-inheritance: -landsat.utils module --------------------- +utils.py ++++++++++++++++++++++ .. automodule:: landsat.utils :members: :undoc-members: :show-inheritance: - - -Module contents ---------------- - -.. automodule:: landsat - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py index 7eed804..0aa35d8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # -# landsat documentation build configuration file, created by -# sphinx-quickstart on Tue Jul 9 22:26:36 2013. +# Landsat-util documentation build configuration file, created by +# sphinx-quickstart on Thu May 28 17:52:10 2015. # # This file is execfile()d with the current directory set to its # containing dir. @@ -14,38 +13,55 @@ # serve to show the default. import sys -import os +from mock import Mock as MagicMock + + +class Mock(MagicMock): + @classmethod + def __getattr__(cls, name): + return Mock() -# If extensions (or modules to document with autodoc) are in another -# directory, add these directories to sys.path here. If the directory is -# relative to the documentation root, use os.path.abspath to make it -# absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +MOCK_MODULES = ['numpy', 'rasterio', 'scipy', 'scikit-image', 'homura', 'boto', + 'termcolor', 'requests', 'python-dateutil'] +sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) + +import os +import sphinx_rtd_theme -# Get the project root dir, which is the parent dir of this +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. cwd = os.getcwd() project_root = os.path.dirname(cwd) -# Insert the project root dir as the first element in the PYTHONPATH. -# This lets us ensure that the source package is imported, and that its -# version is used. sys.path.insert(0, project_root) +print project_root + import landsat -# -- General configuration --------------------------------------------- +# -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.coverage', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] -# The suffix of source filenames. +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The encoding of source files. @@ -55,12 +71,13 @@ master_doc = 'index' # General information about the project. -project = u'A utility to search, download and process Landsat 8 satellite imagery' -copyright = u'2015, Scisco' +project = u'landsat-util' +copyright = u'2015, Development Seed' +author = u'Development Seed' -# The version info for the project you're documenting, acts as replacement -# for |version| and |release|, also used in various other places throughout -# the built documents. +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. # # The short X.Y version. version = landsat.__version__ @@ -69,10 +86,13 @@ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None -# There are two options for replacing |today|: either, you set today to -# some non-false value, then it is used: +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' @@ -102,19 +122,23 @@ # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] -# If true, keep warnings as "system message" paragraphs in the built -# documents. +# If true, keep warnings as "system message" paragraphs in the built documents. #keep_warnings = False +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False -# -- Options for HTML output ------------------------------------------- + +# -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = "sphinx_rtd_theme" + +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] -# Theme options are theme-specific and customize the look and feel of a -# theme further. For a list of options available for each theme, see the +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} @@ -125,27 +149,30 @@ # " v documentation". #html_title = None -# A shorter title for the navigation bar. Default is the same as -# html_title. +# A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None -# The name of an image file (relative to this directory) to place at the -# top of the sidebar. +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. #html_logo = None -# The name of an image file (within the static path) to use as favicon -# of the docs. This file should be a Windows icon file (.ico) being -# 16x16 or 32x32 pixels large. +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. #html_favicon = None -# Add any paths that contain custom static files (such as style sheets) -# here, relative to this directory. They are copied after the builtin -# static files, so a file named "default.css" will overwrite the builtin -# "default.css". +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -# If not '', a 'Last updated on:' timestamp is inserted at every page -# bottom, using the given strftime format. +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to @@ -155,8 +182,8 @@ # Custom sidebar templates, maps document names to template names. #html_sidebars = {} -# Additional templates that should be rendered to pages, maps page names -# to template names. +# Additional templates that should be rendered to pages, maps page names to +# template names. #html_additional_pages = {} # If false, no module index is generated. @@ -171,54 +198,67 @@ # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True -# If true, "Created using Sphinx" is shown in the HTML footer. -# Default is True. +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True -# If true, "(C) Copyright ..." is shown in the HTML footer. -# Default is True. +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True -# If true, an OpenSearch description file will be output, and all pages -# will contain a tag referring to it. The value of this option -# must be the base URL from which the finished HTML is served. +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None -# Output file base name for HTML help builder. -htmlhelp_basename = 'landsatdoc' +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} -# -- Options for LaTeX output ------------------------------------------ +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Landsat-utildoc' + +# -- Options for LaTeX output --------------------------------------------- latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - #'papersize': 'letterpaper', +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', - # The font size ('10pt', '11pt' or '12pt'). - #'pointsize': '10pt', +# Additional stuff for the LaTeX preamble. +#'preamble': '', - # Additional stuff for the LaTeX preamble. - #'preamble': '', +# Latex figure (float) alignment +#'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass -# [howto/manual]). +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'landsat.tex', - u'A utility to search, download and process Landsat 8 satellite imagery', - u'Development Seed', 'manual'), + (master_doc, 'Landsat-util.tex', u'Landsat-util Documentation', + u'Development Seed', 'manual'), ] -# The name of an image file (relative to this directory) to place at -# the top of the title page. +# The name of an image file (relative to this directory) to place at the top of +# the title page. #latex_logo = None -# For "manual" documents, if this is true, then toplevel headings -# are parts, not chapters. +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. #latex_use_parts = False # If true, show page references after internal links. @@ -234,32 +274,28 @@ #latex_domain_indices = True -# -- Options for manual page output ------------------------------------ +# -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'landsat', - u'A utility to search, download and process Landsat 8 satellite imagery', - [u'Development Seed'], 1) + (master_doc, 'landsat-util', u'Landsat-util Documentation', + [author], 1) ] # If true, show URL addresses after external links. #man_show_urls = False -# -- Options for Texinfo output ---------------------------------------- +# -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'landsat', - u'A utility to search, download and process Landsat 8 satellite imagery', - u'Development Seed', - 'landsat', - 'A utility to search, download and process Landsat 8 satellite imagery', - 'Miscellaneous'), + (master_doc, 'Landsat-util', u'Landsat-util Documentation', + author, 'Landsat-util', 'One line description of project.', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. diff --git a/docs/index.rst b/docs/index.rst index ad2a82f..a9516bb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,26 +1,38 @@ -.. csv2es documentation master file, created by - sphinx-quickstart on Tue Jul 9 22:26:36 2013. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +landsat-util +============== -Landsat-util -====================================== +.. image:: https://travis-ci.org/developmentseed/landsat-util.svg?branch=v0.5 + :target: https://travis-ci.org/developmentseed/landsat-util -Contents: +.. image:: https://badge.fury.io/py/landsat-util.svg + :target: http://badge.fury.io/py/landsat-util + +.. image:: https://img.shields.io/pypi/dm/landsat-util.svg + :target: https://pypi.python.org/pypi/landsat-util/ + :alt: Downloads + +.. image:: https://img.shields.io/pypi/l/landsat-util.svg + :target: https://pypi.python.org/pypi/landsat-util/ + :alt: License + +Landsat-util is a command line utility that makes it easy to search, download, and process Landsat imagery. + +This tool uses Development Seed's `API for Landsat Metadata `_. + +This API is accessible here: https://api.developmentseed.org/landsat + +You can also run your own API and connect it to this tool. + +**Table of Contents:** .. toctree:: - :maxdepth: 2 + :maxdepth: 3 - readme installation - usage - contributing - authors - history + overview + todo + api + notes + -Indices and tables -================== -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..3a4ff23 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,71 @@ +Installation +=============== + +Mac OSX +++++++++ + +:: + + $: pip install landsat-util + +Ubuntu 14.10 +++++++++++++ + +Use pip to install landsat-util. If you are not using virtualenv, you might have to run ``pip`` as ``sudo``:: + + $: sudo apt-get update + $: sudo apt-get install python-pip python-numpy python-scipy libgdal-dev libatlas-base-dev gfortran libfreetype6-dev + $: pip install landsat-util + +Other systems ++++++++++++++ + +Make sure Python setuptools is installed:: + + $: python setup.py numpy six + $: python setup.py install + +Docker +++++++ + +If you have docker installed, you can use landsat-util image on docker:: + + $: docker pull developmentseed/landsat-util + $: docker run -it developmentseed/landsat-util:latest /bin/sh -c "landsat -h" + +To use docker version run:: + + $: docker run -it -v ~/landsat:/root/landsat developmentseed/landsat-util:latest landsat -h + +Example commands:: + + $: docker run -it -v ~/landsat:/root/landsat developmentseed/landsat-util:latest landsat search --cloud 4 --start "january 1 2014" --end "january 10 2014" -p 009,045 + $: docker run -it -v ~/landsat:/root/landsat developmentseed/landsat-util:latest landsat download LC80090452014008LGN00 --bands 432 + +This commands mounts ``landsat`` folder in your home directory to ``/root/landsat`` in docker. All downloaded and processed images are stored in ``~/landsat`` folder of your computer. + +If you are using Windows replace ``~/landsat`` with ``/c/Users/``. + + +Upgrade ++++++++ + +:: + + $: pip install -U landsat-util + +If you have installed previous version of landsat using brew, first run:: + + $: brew uninstall landsat-util + +Running Tests ++++++++++++++ + +:: + + $: pip install -U requirements/dev.txt + $: nosetests + +Or:: + + $: python setup.py test diff --git a/docs/landsat.tests.rst b/docs/landsat.tests.rst deleted file mode 100644 index c553762..0000000 --- a/docs/landsat.tests.rst +++ /dev/null @@ -1,62 +0,0 @@ -landsat.tests package -===================== - -Submodules ----------- - -landsat.tests.test_download module ----------------------------------- - -.. automodule:: landsat.tests.test_download - :members: - :undoc-members: - :show-inheritance: - -landsat.tests.test_image module -------------------------------- - -.. automodule:: landsat.tests.test_image - :members: - :undoc-members: - :show-inheritance: - -landsat.tests.test_landsat module ---------------------------------- - -.. automodule:: landsat.tests.test_landsat - :members: - :undoc-members: - :show-inheritance: - -landsat.tests.test_mixins module --------------------------------- - -.. automodule:: landsat.tests.test_mixins - :members: - :undoc-members: - :show-inheritance: - -landsat.tests.test_search module --------------------------------- - -.. automodule:: landsat.tests.test_search - :members: - :undoc-members: - :show-inheritance: - -landsat.tests.test_utils module -------------------------------- - -.. automodule:: landsat.tests.test_utils - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: landsat.tests - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/make.bat b/docs/make.bat index 08cb1de..39b62e5 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -1,242 +1,263 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\landsat.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\landsat.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) - -:end +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + echo. coverage to run coverage check of the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +REM Check if sphinx-build is available and fallback to Python version if any +%SPHINXBUILD% 2> nul +if errorlevel 9009 goto sphinx_python +goto sphinx_ok + +:sphinx_python + +set SPHINXBUILD=python -m sphinx.__init__ +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +:sphinx_ok + + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Landsat-util.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Landsat-util.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "coverage" ( + %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage + if errorlevel 1 exit /b 1 + echo. + echo.Testing of coverage in the sources finished, look at the ^ +results in %BUILDDIR%/coverage/python.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/docs/modules.rst b/docs/modules.rst deleted file mode 100644 index 147141d..0000000 --- a/docs/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -landsat -======= - -.. toctree:: - :maxdepth: 4 - - landsat diff --git a/docs/notes.rst b/docs/notes.rst new file mode 100644 index 0000000..cabe864 --- /dev/null +++ b/docs/notes.rst @@ -0,0 +1,10 @@ +Important Notes +=============== + +- All downloaded and processed images are stored at your home directory in landsat forlder: ``~/landsat`` + +- The image thumbnail web address that is included in the results can be used to make sure that clouds are not obscuring the subject of interest. Run the search again if you need to narrow down your result and then start downloading images. Each image is usually more than 700mb and it might takes a very long time if there are too many images to download + +- Image processing is a very heavy and resource consuming task. Each process takes about 5-10 mins. We recommend that you run the processes in smaller badges. Pansharpening, while increasing image resolution 2x, substantially increases processing time. + +- Landsat-util requires at least 2GB of Memory (RAM). diff --git a/docs/overview.rst b/docs/overview.rst new file mode 100644 index 0000000..d0ce79f --- /dev/null +++ b/docs/overview.rst @@ -0,0 +1,82 @@ +Overview: What can landsat-util do? +==================================== + +Landsat-util has three main functions: + +- **Search** for landsat tiles based on several search parameters. +- **Download** landsat images. +- **Image processing** and pan sharpening on landsat images. + +These three functions have to be performed separately. + +**Help**: Type ``landsat -h`` for detailed usage parameters. + +Search +++++++ + +Search returns information about all landsat tiles that match your criteria. This includes a link to an unprocessed preview of the tile. The most important result is the tile's *sceneID*, which you will need to download the tile (see step 2 below). + +Search for landsat tiles in a given geographical region, using any of the following: + +- **Paths and rows**: If you know the paths and rows you want to search for. +- **Latidue and Longitude**: If you need the latitude and longitude of the point you want to search for. + +Additionally filter your search using the following parameters: + +- **Start and end dates** for when imagery was taken +- **Maximum percent cloud cover** (default is 20%) + +**Examples of search**: + +Search by path and row:: + + $: landsat search --cloud 4 --start "january 1 2014" --end "january 10 2014" -p 009,045 + +Search by latitude and longitude:: + + $: landsat search --lat 38.9004204 --lon -77.0237117 + + +Download +++++++++ + +You can download tiles using their unique sceneID, which you get from landsat search. + +Landsat-util will download a zip file that includes all the bands. You have the option of specifying the bands you want to down. In this case, landsat-util only downloads those bands if they are available online. + +**Examples of download**: + +Download images by their custom sceneID, which you get from landsat search:: + + $: landsat download LC80090452014008LGN00 + +Download only band 4, 3 and 2 for a particular sceneID:: + + $: landsat download LC80090452014008LGN00 --bands 432 + +Download multiple sceneIDs:: + + $: landsat download LC80090452014008LGN00 LC80090452015008LGN00 LC80090452013008LGN00 + +Image processing +++++++++++++++++ + +You can process your downloaded tiles with our custom image processing algorithms. In addition, you can choose to pansharpen your images and specify which bands to process. + +**Examples of image processing**: + +Process images that are already downloaded. Remember, the program accepts both zip files and unzipped folders:: + + $: landsat process path/to/LC80090452014008LGN00.tar.bz + +If unzipped:: + + $: landsat process path/to/LC80090452014008LGN00 + +Specify bands 3, 5 and 1:: + + $: landsat process path/to/LC80090452014008LGN00 --bands 351 + +Process *and* pansharpen a downloaded image:: + + $: landsat process path/to/LC80090452014008LGN00.tar.bz --pansharpen diff --git a/docs/todo.rst b/docs/todo.rst new file mode 100644 index 0000000..afb754d --- /dev/null +++ b/docs/todo.rst @@ -0,0 +1,9 @@ +To Do List +++++++++++ + +- Add Sphinx Documentation +- Add capacity for NDVI output +- Add alternative projections (currently only option is default web-mercator; EPSG: 3857) +- Connect search to Google Address API +- Include 16-bit image variant in output +- Add support for color correct looping over multiple compressed inputs (currently just 1) diff --git a/landsat/__init__.py b/landsat/__init__.py index a68d2bd..a71c5c7 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -1 +1 @@ -__version__ = '0.6.3' +__version__ = '0.7.0' diff --git a/landsat/downloader.py b/landsat/downloader.py index 101d73a..10bbc6e 100644 --- a/landsat/downloader.py +++ b/landsat/downloader.py @@ -11,14 +11,17 @@ class RemoteFileDoesntExist(Exception): + """ Exception to be used when the remote file does not exist """ pass class IncorrectSceneId(Exception): + """ Exception to be used when scene id is incorrect """ pass class Downloader(VerbosityMixin): + """ The downloader class """ def __init__(self, verbose=False, download_dir=None): self.download_dir = download_dir if download_dir else settings.DOWNLOAD_DIR @@ -32,12 +35,22 @@ def download(self, scenes, bands=None): """ Download scenese from Google Storage or Amazon S3 if bands are provided - @params - scenes - A list of sceneIDs - bands - A list of bands + :param scenes: + A list of scene IDs + :type scenes: + List + :param bands: + A list of bands. Default value is None. + :type scenes: + List + + :returns: + (List) includes downloaded scenes as key and source as value (aws or google) """ if isinstance(scenes, list): + output = {} + for scene in scenes: # If bands are provided the image is from 2015 or later use Amazon if (bands and int(scene[12]) > 4): @@ -50,20 +63,38 @@ def download(self, scenes, bands=None): bands_plus.append('MTL') for band in bands_plus: self.amazon_s3(scene, band, path) + output[scene] = 'aws' except RemoteFileDoesntExist: self.google_storage(scene, self.download_dir) + output[scene] = 'google' + else: raise Exception('Expected bands list') else: self.google_storage(scene, self.download_dir) + output[scene] = 'google' - return True + return output else: raise Exception('Expected sceneIDs list') def google_storage(self, scene, path): - """ Google Storage Downloader """ + """ + Google Storage Downloader. + + :param scene: + The scene id + :type scene: + String + :param path: + The directory path to where the image should be stored + :type path: + String + + :returns: + Boolean + """ sat = self.scene_interpreter(scene) filename = scene + '.tar.bz' @@ -76,7 +107,25 @@ def google_storage(self, scene, path): raise RemoteFileDoesntExist('%s is not available on Google Storage' % filename) def amazon_s3(self, scene, band, path): - """ Amazon S3 downloader """ + """ + Amazon S3 downloader + + :param scene: + The scene ID. + :type scene: + String + :param band: + The band number. + :type band: + String, Integer + :param path: + The directory path to where the image should be stored + :type path: + String + + :returns: + Boolean + """ sat = self.scene_interpreter(scene) if band != 'MTL': @@ -92,6 +141,24 @@ def amazon_s3(self, scene, band, path): raise RemoteFileDoesntExist('%s is not available on Amazon S3' % filename) def fetch(self, url, path, filename): + """ Downloads the given url. + + :param url: + The url to be downloaded. + :type url: + String + :param path: + The directory path to where the image should be stored + :type path: + String + :param filename: + The filename that has to be downloaded + :type filename: + String + + :returns: + Boolean + """ self.output('Downloading: %s' % filename, normal=True, arrow=True) @@ -108,22 +175,48 @@ def fetch(self, url, path, filename): def google_storage_url(self, sat): """ - Return a google storage url the contains the scene provided - @params - sat - expects an object created by scene_interpreter method + Returns a google storage url the contains the scene provided. + + :param sat: + Expects an object created by scene_interpreter method + :type sat: + dict + + :returns: + (String) The URL to a google storage file """ filename = sat['scene'] + '.tar.bz' return join(self.google, sat['sat'], sat['path'], sat['row'], filename) def amazon_s3_url(self, sat, filename): """ - Return an amazon s3 url the contains the scene and band provided - @params - sat - expects an object created by scene_interpreter method + Return an amazon s3 url the contains the scene and band provided. + + :param sat: + Expects an object created by scene_interpreter method + :type sat: + dict + :param filename: + The filename that has to be downloaded from Amazon + :type filename: + String + + :returns: + (String) The URL to a S3 file """ return join(self.s3, sat['sat'], sat['path'], sat['row'], sat['scene'], filename) def remote_file_exists(self, url): + """ Checks whether the remote file exists. + + :param url: + The url that has to be checked. + :type url: + String + + :returns: + **True** if remote file exists and **False** if it doesn't exist. + """ status = requests.head(url).status_code if status == 200: @@ -132,12 +225,39 @@ def remote_file_exists(self, url): return False def get_remote_file_size(self, url): - """ Gets the filesize of a remote file """ + """ Gets the filesize of a remote file. + + :param url: + The url that has to be checked. + :type url: + String + + :returns: + int + """ headers = requests.head(url).headers return int(headers['content-length']) def scene_interpreter(self, scene): - """ Conver sceneID to rows, paths and dates """ + """ Conver sceneID to rows, paths and dates. + + :param scene: + The scene ID. + :type scene: + String + + :returns: + dict + + :Example output: + + >>> anatomy = { + 'path': None, + 'row': None, + 'sat': None, + 'scene': scene + } + """ anatomy = { 'path': None, 'row': None, diff --git a/landsat/image.py b/landsat/image.py index fd7013e..0a7d928 100644 --- a/landsat/image.py +++ b/landsat/image.py @@ -4,7 +4,7 @@ import warnings import sys -from os.path import join +from os.path import join, isdir import tarfile import glob import subprocess @@ -23,24 +23,40 @@ class FileDoesNotExist(Exception): + """ Exception to be used when the file does not exist. """ pass class Process(VerbosityMixin): """ Image procssing class + + To initiate the following parameters must be passed: + + :param path: + Path of the image. + :type path: + String + :param bands: + The band sequence for the final image. Must be a python list. (optional) + :type bands: + List + :param dst_path: + Path to the folder where the image should be stored. (optional) + :type dst_path: + String + :param verbose: + Whether the output should be verbose. Default is False. + :type verbose: + boolean + :param force_unzip: + Whether to force unzip the tar file. Default is False + :type force_unzip: + boolean + """ - def __init__(self, path, bands=None, dst_path=None, verbose=False): - """ - @params - scene - the scene ID - bands - The band sequence for the final image. Must be a python list - src_path - The path to the source image bundle - dst_path - The destination path - zipped - Set to true if the scene is in zip format and requires unzipping - verbose - Whether to sh ow verbose output - """ + def __init__(self, path, bands=None, dst_path=None, verbose=False, force_unzip=False): self.projection = {'init': 'epsg:3857'} self.dst_crs = {'init': u'epsg:3857'} @@ -59,13 +75,23 @@ def __init__(self, path, bands=None, dst_path=None, verbose=False): self.scene_path = join(self.src_path, self.scene) if self._check_if_zipped(path): - self._unzip(join(self.src_path, get_file(path)), join(self.src_path, self.scene), self.scene) + self._unzip(join(self.src_path, get_file(path)), join(self.src_path, self.scene), self.scene, force_unzip) self.bands_path = [] for band in self.bands: self.bands_path.append(join(self.scene_path, self._get_full_filename(band))) def run(self, pansharpen=True): + """ Executes the image processing. + + :param pansharpen: + Whether the process should also run pansharpenning. Default is True + :type pansharpen: + boolean + + :returns: + (String) the path to the processed image + """ self.output("* Image processing started for bands %s" % "-".join(map(str, self.bands)), normal=True) @@ -120,10 +146,8 @@ def run(self, pansharpen=True): dst_shape = src_data['shape'] dst_corner_ys = [crn[k]['y'][1][0] for k in crn.keys()] dst_corner_xs = [crn[k]['x'][1][0] for k in crn.keys()] - y_pixel = abs(max(dst_corner_ys) - - min(dst_corner_ys)) / dst_shape[0] - x_pixel = abs(max(dst_corner_xs) - - min(dst_corner_xs)) / dst_shape[1] + y_pixel = abs(max(dst_corner_ys) - min(dst_corner_ys)) / dst_shape[0] + x_pixel = abs(max(dst_corner_xs) - min(dst_corner_xs)) / dst_shape[1] dst_transform = (min(dst_corner_xs), x_pixel, @@ -267,14 +291,19 @@ def _get_boundaries(self, src): def _percent_cut(self, color, low, high): return numpy.percentile(color[numpy.logical_and(color > 0, color < 65535)], (low, high)) - def _unzip(self, src, dst, scene): + def _unzip(self, src, dst, scene, force_unzip=False): """ Unzip tar files """ self.output("Unzipping %s - It might take some time" % scene, normal=True, arrow=True) try: - tar = tarfile.open(src, 'r') - tar.extractall(path=dst) - tar.close() + # check if file is already unzipped, skip + if isdir(dst) and not force_unzip: + self.output("%s is already unzipped." % scene, normal=True, arrow=True) + return + else: + tar = tarfile.open(src, 'r') + tar.extractall(path=dst) + tar.close() except tarfile.ReadError: check_create_folder(dst) subprocess.check_call(['tar', '-xf', src, '-C', dst]) diff --git a/landsat/landsat.py b/landsat/landsat.py index c81f4d2..bb034df 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -86,6 +86,8 @@ --region URL to S3 region e.g. s3-us-west-2.amazonaws.com + --force-unzip Force unzip tar file + Process: landsat.py process path [-h] [-b --bands] [-p --pansharpen] @@ -115,10 +117,18 @@ --bucket Bucket name (required if uploading to s3) --region URL to S3 region e.g. s3-us-west-2.amazonaws.com + + --force-unzip Force unzip tar file """ def args_options(): + """ Generates an arugment parser. + + :returns: + Parser object + """ + parser = argparse.ArgumentParser(prog='landsat', formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent(DESCRIPTION)) @@ -173,6 +183,7 @@ def args_options(): 'as Environment Variables)') parser_download.add_argument('--bucket', help='Bucket name (required if uploading to s3)') parser_download.add_argument('--region', help='URL to S3 region e.g. s3-us-west-2.amazonaws.com') + parser_download.add_argument('--force-unzip', help='Force unzip tar file', action='store_true') parser_process = subparsers.add_parser('process', help='Process Landsat imagery') parser_process.add_argument('path', @@ -192,21 +203,35 @@ def args_options(): 'as Environment Variables)') parser_process.add_argument('--bucket', help='Bucket name (required if uploading to s3)') parser_process.add_argument('--region', help='URL to S3 region e.g. s3-us-west-2.amazonaws.com') + parser_process.add_argument('--force-unzip', help='Force unzip tar file', action='store_true') return parser def main(args): """ - Main function - launches the program + Main function - launches the program. + + :param args: + The Parser arguments + :type args: + Parser object + + :returns: + List + + :example: + >>> ["The latitude and longitude values must be valid numbers", 1] """ v = VerbosityMixin() if args: + if args.subs == 'process': verbose = True if args.verbose else False - stored = process_image(args.path, args.bands, verbose, args.pansharpen) + force_unzip = True if args.force_unzip else False + stored = process_image(args.path, args.bands, verbose, args.pansharpen, force_unzip) if args.upload: u = Uploader(args.key, args.secret, args.region) @@ -252,18 +277,25 @@ def main(args): elif args.subs == 'download': d = Downloader(download_dir=args.dest) try: - if d.download(args.scenes, convert_to_integer_list(args.bands)): - if args.process: + bands = convert_to_integer_list(args.bands) + if args.pansharpen: + bands.append(8) + + downloaded = d.download(args.scenes, bands) + + if args.process: + force_unzip = True if args.force_unzip else False + for scene, src in downloaded.iteritems(): if args.dest: - path = join(args.dest, args.scenes[0]) + path = join(args.dest, scene) else: - path = join(settings.DOWNLOAD_DIR, args.scenes[0]) + path = join(settings.DOWNLOAD_DIR, scene) # Keep using Google if the image is before 2015 - if (int(args.scenes[0][12]) < 5 or not args.bands): + if src == 'google': path = path + '.tar.bz' - stored = process_image(path, args.bands, False, args.pansharpen) + stored = process_image(path, args.bands, False, args.pansharpen, force_unzip) if args.upload: try: @@ -274,17 +306,41 @@ def main(args): return ["Connection timeout. Probably the region parameter is incorrect", 1] u.run(args.bucket, get_file(stored), stored) - return ["The output is stored at %s" % stored] - else: - return ['Download Completed', 0] + v.output("The output is stored at %s" % stored, normal=True, arrow=True) + + return ['Image Processing Completed', 0] + else: + return ['Download Completed', 0] except IncorrectSceneId: return ['The SceneID provided was incorrect', 1] -def process_image(path, bands=None, verbose=False, pansharpen=False): +def process_image(path, bands=None, verbose=False, pansharpen=False, force_unzip=None): + """ Handles constructing and image process. + + :param path: + The path to the image that has to be processed + :type path: + String + :param bands: + List of bands that has to be processed. (optional) + :type bands: + List + :param verbose: + Sets the level of verbosity. Default is False. + :type verbose: + boolean + :param pansharpen: + Whether to pansharpen the image. Default is False. + :type pansharpen: + boolean + + :returns: + (String) path to the processed image + """ try: bands = convert_to_integer_list(bands) - p = Process(path, bands=bands, verbose=verbose) + p = Process(path, bands=bands, verbose=verbose, force_unzip=force_unzip) except IOError: exit("Zip file corrupted", 1) except FileDoesNotExist as e: diff --git a/landsat/mixins.py b/landsat/mixins.py index a50d8b4..0adbf66 100644 --- a/landsat/mixins.py +++ b/landsat/mixins.py @@ -10,9 +10,6 @@ class VerbosityMixin(object): """ Verbosity Mixin that generates beautiful stdout outputs. - - Main method: - output() """ verbose = False @@ -24,14 +21,33 @@ def output(self, value, normal=False, color=None, error=False, if class instance verbose is True, the value is printed - @param - - value: (string) the message to be printed - - nomral: (boolean) if set to true the message is always printed, - otherwise it is only shown if verbosity is set - - color: (string) The color of the message, choices: 'red', 'green', 'blue' - - error: (boolean) if set to true the message appears in red - - arrow: (boolean) if set to true an arrow appears before the message - - indent: (integer) indents the message based on the number provided + :param value: + a string representing the message to be printed + :type value: + String + :param normal: + if set to true the message is always printed, otherwise it is only shown if verbosity is set + :type normal: + boolean + :param color: + The color of the message, choices: 'red', 'green', 'blue' + :type normal: + String + :param error: + if set to true the message appears in red + :type error: + Boolean + :param arrow: + if set to true an arrow appears before the message + :type arrow: + Boolean + :param indent: + indents the message based on the number provided + :type indent: + Boolean + + :returns: + void """ if error and value and (normal or self.verbose): @@ -44,8 +60,16 @@ def output(self, value, normal=False, color=None, error=False, def subprocess(self, argv): """ - Execute subprocess commands with proper ouput + Execute subprocess commands with proper ouput. This is no longer used in landsat-util + + :param argv: + A list of subprocess arguments + :type argv: + List + + :returns: + void """ if self.verbose: @@ -59,13 +83,22 @@ def subprocess(self, argv): return def exit(self, message): - """ Print an exist message and exit """ + """ outputs an exit message and exits + + :param message: + The message to be outputed + :type message: + String + + :returns: + void + """ self.output(message, normal=True, color="green") sys.exit() def _print(self, msg, color=None, arrow=False, indent=None): - """ Print the msg with the color provided """ + """ Print the msg with the color provided. """ if color: msg = colored(msg, color) diff --git a/landsat/search.py b/landsat/search.py index c3d6bfd..65ebc46 100644 --- a/landsat/search.py +++ b/landsat/search.py @@ -10,6 +10,7 @@ class Search(object): + """ The search class """ def __init__(self): self.api_url = settings.API_URL @@ -17,43 +18,64 @@ def __init__(self): def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None, cloud_min=None, cloud_max=None, limit=1): """ - The main method of Search class. It searches the DevSeed Landsat API - - Returns python dictionary - - Arguments: - paths_rows -- A string in this format: "003,003,004,004". Must be in pairs - lat -- The latitude - lon -- The longitude - start_date -- date string. format: YYYY-MM-DD - end_date -- date string. format: YYYY-MM-DD - cloud_min -- float specifying the minimum percentage. e.g. 4.3 - cloud_max -- float specifying the maximum percentage. e.g. 78.9 - limit -- integer specigying the maximum results return. - - Example: - - search('003,003', '2014-01-01', '2014-06-01') - - will return: - - { - 'status': u'SUCCESS', - 'total_returned': 1, - 'total': 1, - 'limit': 1 - 'results': [ - { - 'sat_type': u'L8', - 'sceneID': u'LC80030032014142LGN00', - 'date': u'2014-05-22', - 'path': u'003', - 'thumbnail': u'http://....../landsat_8/2014/003/003/LC80030032014142LGN00.jpg', - 'cloud': 33.36, - 'row': u'003 - } - ] - } + The main method of Search class. It searches Development Seed's Landsat API. + + :param paths_rows: + A string in this format: "003,003,004,004". Must be in pairs and separated by comma. + :type paths_rows: + String + :param lat: + The latitude + :type lat: + String, float, integer + :param lon: + The The longitude + :type lon: + String, float, integer + :param start_date: + Date string. format: YYYY-MM-DD + :type start_date: + String + :param end_date: + date string. format: YYYY-MM-DD + :type end_date: + String + :param cloud_min: + float specifying the minimum percentage. e.g. 4.3 + :type cloud_min: + float + :param cloud_max: + float specifying the maximum percentage. e.g. 78.9 + :type cloud_max: + float + :param limit: + integer specigying the maximum results return. + :type limit: + integer + + :returns: + dict + + :example: + >>> search = Search() + >>> search('003,003', '2014-01-01', '2014-06-01') + >>> { + 'status': u'SUCCESS', + 'total_returned': 1, + 'total': 1, + 'limit': 1 + 'results': [ + { + 'sat_type': u'L8', + 'sceneID': u'LC80030032014142LGN00', + 'date': u'2014-05-22', + 'path': u'003', + 'thumbnail': u'http://....../landsat_8/2014/003/003/LC80030032014142LGN00.jpg', + 'cloud': 33.36, + 'row': u'003 + } + ] + } """ search_string = self.query_builder(paths_rows, lat, lon, start_date, end_date, cloud_min, cloud_max) @@ -89,7 +111,40 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date= def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None, cloud_min=None, cloud_max=None): - """ Builds the proper search syntax (query) for Landsat API """ + """ Builds the proper search syntax (query) for Landsat API. + + :param paths_rows: + A string in this format: "003,003,004,004". Must be in pairs and separated by comma. + :type paths_rows: + String + :param lat: + The latitude + :type lat: + String, float, integer + :param lon: + The The longitude + :type lon: + String, float, integer + :param start_date: + Date string. format: YYYY-MM-DD + :type start_date: + String + :param end_date: + date string. format: YYYY-MM-DD + :type end_date: + String + :param cloud_min: + float specifying the minimum percentage. e.g. 4.3 + :type cloud_min: + float + :param cloud_max: + float specifying the maximum percentage. e.g. 78.9 + :type cloud_max: + float + + :returns: + String + """ query = [] or_string = '' @@ -131,15 +186,37 @@ def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, en def row_path_builder(self, path='', row=''): """ - Builds row and path query - Accepts row and path in XXX format, e.g. 003 + Builds row and path query. + + :param path: + Landsat path. Must be three digits + :type path: + String + :param row: + Landsat row. Must be three digits + :type row: + String + + :returns: + String """ return 'path:%s+AND+row:%s' % (path, row) def date_range_builder(self, start='2013-02-11', end=None): """ - Builds date range query - Accepts start and end date in this format YYYY-MM-DD + Builds date range query. + + :param start: + Date string. format: YYYY-MM-DD + :type start: + String + :param end: + date string. format: YYYY-MM-DD + :type end: + String + + :returns: + String """ if not end: end = time.strftime('%Y-%m-%d') @@ -148,13 +225,37 @@ def date_range_builder(self, start='2013-02-11', end=None): def cloud_cover_prct_range_builder(self, min=0, max=100): """ - Builds cloud cover percentage range query - Accepts bottom and top range in float, e.g. 1.00 + Builds cloud cover percentage range query. + + :param min: + float specifying the minimum percentage. Default is 0 + :type min: + float + :param max: + float specifying the maximum percentage. Default is 100 + :type max: + float + + :returns: + String """ return 'cloudCoverFull:[%s+TO+%s]' % (min, max) def lat_lon_builder(self, lat=0, lon=0): - """ Builds lat and lon query """ + """ Builds lat and lon query. + + :param lat: + The latitude. Default is 0 + :type lat: + float + :param lon: + The The longitude. Default is 0 + :type lon: + float + + :returns: + String + """ return ('upperLeftCornerLatitude:[%s+TO+1000]+AND+lowerRightCornerLatitude:[-1000+TO+%s]' '+AND+lowerLeftCornerLongitude:[-1000+TO+%s]+AND+upperRightCornerLongitude:[%s+TO+1000]' % (lat, lat, lon, lon)) diff --git a/landsat/tests/test_download.py b/landsat/tests/test_download.py index 6166497..4d7d06d 100644 --- a/landsat/tests/test_download.py +++ b/landsat/tests/test_download.py @@ -28,7 +28,9 @@ def setUpClass(cls): cls.d = Downloader() cls.temp_folder = mkdtemp() cls.scene = 'LT81360082013127LGN01' + cls.scene_2 = 'LC82050312014229LGN00' cls.scene_s3 = 'LC80010092015051LGN00' + cls.scene_s3_2 = 'LC82050312015136LGN00' cls.scene_size = 59204484 @classmethod @@ -49,21 +51,19 @@ def assertSize(self, url, path): def test_download(self, mock_fetch): mock_fetch.return_value = True - # download one list + # download one scene self.d.download([self.scene]) - self.assertTrue(self.d.download([self.scene])) + self.assertEqual({self.scene: 'google'}, self.d.download([self.scene])) + + # download multiple scenes + self.assertEqual({self.scene: 'google', self.scene_2: 'google'}, self.d.download([self.scene, self.scene_2])) # Test if error is raised when passing scene as string instead of list self.assertRaises(Exception, self.d.download, self.scene) - # Test if download works when passing scenes as list - self.d.download([self.scene, self.scene]) - self.assertTrue(self.d.download([self.scene])) - # Test when passing band list along with sceneID - self.d.download([self.scene_s3], bands=[11]) - - self.assertTrue(self.d.download([self.scene])) + self.assertEqual({self.scene_s3: 'aws', self.scene_s3_2: 'aws'}, + self.d.download([self.scene_s3, self.scene_s3_2], bands=[11])) # Test whether passing band as string raises an exception self.assertRaises(Exception, self.d.download, self.scene, 4) diff --git a/landsat/tests/test_landsat.py b/landsat/tests/test_landsat.py index d6a74c1..6d8d9a3 100644 --- a/landsat/tests/test_landsat.py +++ b/landsat/tests/test_landsat.py @@ -94,21 +94,38 @@ def test_download_incorrect(self): @mock.patch('landsat.landsat.Downloader.download') def test_download_process_continuous(self, mock_downloader, mock_process): """Test download and process commands together""" - mock_downloader.return_value = True + mock_downloader.return_value = {'LC80010092015051LGN00': 'aws', + 'LC80010092014051LGN00': 'aws'} mock_process.return_value = 'image.TIF' - args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.mock_path, '-p'] + args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', self.mock_path, '-p'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2']) - mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False) - self.assertEquals(output, ["The output is stored at image.TIF"]) + mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], ['4', '3', '2']) + mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, False) + self.assertEquals(output, ["Image Processing Completed", 0]) + + # Call with force unzip flag + args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', + self.mock_path, '-p', '--force-unzip'] + output = landsat.main(self.parser.parse_args(args)) + mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], ['4', '3', '2']) + mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, True) + self.assertEquals(output, ["Image Processing Completed", 0]) + + # Call with pansharpen + args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', + self.mock_path, '-p', '--pansharpen'] + output = landsat.main(self.parser.parse_args(args)) + mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], ['4', '3', '2', 8]) + mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, True, False) + self.assertEquals(output, ["Image Processing Completed", 0]) @mock.patch('landsat.landsat.Uploader') @mock.patch('landsat.landsat.process_image') @mock.patch('landsat.landsat.Downloader.download') def test_download_process_continuous_with_upload(self, mock_downloader, mock_process, mock_upload): """Test download and process commands together""" - mock_downloader.return_value = True + mock_downloader.return_value = {'LC80010092015051LGN00': 'aws'} mock_process.return_value = 'image.TIF' mock_upload.run.return_value = True @@ -116,23 +133,23 @@ def test_download_process_continuous_with_upload(self, mock_downloader, mock_pro '-u', '--key', 'somekey', '--secret', 'somesecret', '--bucket', 'mybucket', '--region', 'this'] output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2']) - mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False) mock_upload.assert_called_with('somekey', 'somesecret', 'this') mock_upload.return_value.run.assert_called_with('mybucket', 'image.TIF', 'image.TIF') - self.assertEquals(output, ["The output is stored at image.TIF"]) + self.assertEquals(output, ["Image Processing Completed", 0]) @mock.patch('landsat.landsat.process_image') @mock.patch('landsat.landsat.Downloader.download') def test_download_process_continuous_with_wrong_args(self, mock_downloader, mock_process): """Test download and process commands together""" - mock_downloader.return_value = True + mock_downloader.return_value = {'LC80010092015051LGN00': 'aws'} mock_process.return_value = 'image.TIF' args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.mock_path, '-p', '-u', '--region', 'whatever'] output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2']) - mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False) self.assertEquals(output, ['Could not authenticate with AWS', 1]) @mock.patch('landsat.landsat.process_image') @@ -143,7 +160,7 @@ def test_process_correct(self, mock_process): args = ['process', 'path/to/folder/LC80010092015051LGN00'] output = landsat.main(self.parser.parse_args(args)) - mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, False) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, False, False) self.assertEquals(output, ["The output is stored at image.TIF"]) @mock.patch('landsat.landsat.process_image') @@ -154,7 +171,7 @@ def test_process_correct_pansharpen(self, mock_process): args = ['process', '--pansharpen', 'path/to/folder/LC80010092015051LGN00'] output = landsat.main(self.parser.parse_args(args)) - mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, True) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, True, False) self.assertEquals(output, ["The output is stored at image.TIF"]) def test_process_incorrect(self): diff --git a/landsat/uploader.py b/landsat/uploader.py index bbbc151..c1e9e4f 100644 --- a/landsat/uploader.py +++ b/landsat/uploader.py @@ -27,6 +27,25 @@ class Uploader(VerbosityMixin): + """ + The Uploader class. + + To initiate the following parameters must be passed: + + :param key: + AWS access key id (optional) + :type key: + String + :param secret: + AWS access secret key (optional) + :type secret: + String + :param host: + AWS host, e.g. s3.amazonaws.com (optional) + :type host: + String + """ + progress_template = \ 'File Size:%(size)4d MB | Uploaded:%(uploaded)4d MB' + ' ' * 8 @@ -37,6 +56,25 @@ def __init__(self, key=None, secret=None, host=None): self.conn = S3Connection(key, secret, host=host) def run(self, bucket_name, filename, path): + """ + Initiate the upload. + + :param bucket_name: + Name of the S3 bucket + :type bucket_name: + String + :param filename: + The filname + :type filename: + String + :param path: + The path to the file that needs to be uploaded + :type path: + String + + :returns: + void + """ f = open(path, 'rb') self.source_size = os.stat(path).st_size @@ -66,15 +104,16 @@ def cb(part_no, uploaded, total): def data_collector(iterable, def_buf_size=5242880): - ''' Buffers n bytes of data + """ Buffers n bytes of data. - Args: - iterable: could be a list, generator or string - def_buf_size: number of bytes to buffer, default is 5mb + :param iterable: + Could be a list, generator or string + :type iterable: + List, generator, String - Returns: - A generator object - ''' + :returns: + A generator object + """ buf = '' for data in iterable: buf += data @@ -108,23 +147,54 @@ def upload(bucket, aws_access_key, aws_secret_key, iterable, key, progress_cb=None, threads=5, replace=False, secure=True, connection=None): - ''' Upload data to s3 using the s3 multipart upload API. - - Args: - bucket: name of s3 bucket - aws_access_key: aws access key - aws_secret_key: aws secret key - iterable: The data to upload. Each 'part' in the list - will be uploaded in parallel. Each part must be at - least 5242880 bytes (5mb). - key: the name of the key to create in the s3 bucket - progress_cb: will be called with (part_no, uploaded, total) - each time a progress update is available. - threads: the number of threads to use while uploading. (Default is 5) - replace: will replace the key in s3 if set to true. (Default is false) - secure: use ssl when talking to s3. (Default is true) - connection: used for testing - ''' + """ Upload data to s3 using the s3 multipart upload API. + + :param bucket: + Name of the S3 bucket + :type bucket: + String + :param aws_access_key: + AWS access key id (optional) + :type aws_access_key: + String + :param aws_secret_key: + AWS access secret key (optional) + :type aws_secret_key: + String + :param iterable: + The data to upload. Each 'part' in the list. will be uploaded in parallel. Each part must be at + least 5242880 bytes (5mb). + :type iterable: + An iterable object + :param key: + The name of the key (filename) to create in the s3 bucket + :type key: + String + :param progress_cb: + Progress callback, will be called with (part_no, uploaded, total) each time a progress update + is available. (optional) + :type progress_cb: + function + :param threads: + the number of threads to use while uploading. (Default is 5) + :type threads: + int + :param replace: + will replace the key (filename) on S3 if set to true. (Default is false) + :type replace: + boolean + :param secure: + Use ssl when talking to s3. (Default is true) + :type secure: + boolean + :param connection: + Used for testing (optional) + :type connection: + S3 connection class + + :returns: + void + """ if not connection: from boto.s3.connection import S3Connection as connection diff --git a/landsat/utils.py b/landsat/utils.py index 9591f0b..b5dc6e3 100644 --- a/landsat/utils.py +++ b/landsat/utils.py @@ -12,7 +12,13 @@ class Capturing(list): - """ Captures a subprocess stdout """ + """ + Captures a subprocess stdout. + + :Usage: + >>> with Capturing(): + ... subprocess(args) + """ def __enter__(self): self._stdout = sys.stdout sys.stdout = self._stringio = StringIO() @@ -25,11 +31,11 @@ def __exit__(self, *args): class timer(object): """ - A time class + A timer class. - Usage: - with timer(): - your code + :Usage: + >>> with timer(): + ... your code """ def __enter__(self): self.start = time.time() @@ -40,6 +46,21 @@ def __exit__(self, type, value, traceback): def exit(message, code=0): + """ output a message to stdout and terminates the process. + + :param message: + Message to be outputed. + :type message: + String + :param code: + The termination code. Default is 0 + :type code: + int + + :returns: + void + """ + v = VerbosityMixin() if code == 0: v.output(message, normal=True, arrow=True) @@ -50,13 +71,20 @@ def exit(message, code=0): def create_paired_list(value): - """ Create a list of paired items from a string + """ Create a list of paired items from a string. + + :param value: + the format must be 003,003,004,004 (commas with no space) + :type value: + String - Arguments: - i - the format must be 003,003,004,004 (commas with no space) + :returns: + List - Returns: + :example: + >>> create_paired_list('003,003,004,004') [['003','003'], ['004', '004']] + """ if isinstance(value, list): @@ -75,8 +103,15 @@ def create_paired_list(value): def check_create_folder(folder_path): - """ Check whether a folder exists, if not the folder is created - Always return folder_path + """ Check whether a folder exists, if not the folder is created. + + :param folder_path: + Path to the folder + :type folder_path: + String + + :returns: + (String) the path to the folder """ if not os.path.exists(folder_path): os.makedirs(folder_path) @@ -85,20 +120,55 @@ def check_create_folder(folder_path): def get_file(path): - """ Separate the name of the file or folder from the path and return it - Example: /path/to/file ---> file + """ Separate the name of the file or folder from the path and return it. + + :param path: + Path to the folder + :type path: + String + + :returns: + (String) the filename + + :example: + >>> get_file('/path/to/file.jpg') + 'file.jpg' """ return os.path.basename(path) def get_filename(path): - """ Return the filename without extension. e.g. index.html --> index """ + """ Return the filename without extension. + + :param path: + Path to the folder + :type path: + String + + :returns: + (String) the filename without extension + + :example: + >>> get_filename('/path/to/file.jpg') + 'file' + """ return os.path.splitext(get_file(path))[0] def three_digit(number): """ Add 0s to inputs that their length is less than 3. - For example: 1 --> 001 | 02 --> 020 | st --> 0st + + :param number: + The number to convert + :type number: + int + + :returns: + String + + :example: + >>> three_digit(1) + '001' """ number = str(number) if len(number) == 1: @@ -110,8 +180,19 @@ def three_digit(number): def georgian_day(date): - """ Returns the number of days passed since the start of the year - Accepted format: %m/%d/%Y + """ Returns the number of days passed since the start of the year. + + :param date: + The string date with this format %m/%d/%Y + :type date: + String + + :returns: + int + + :example: + >>> georgian_day('05/1/2015') + 121 """ try: fmt = '%m/%d/%Y' @@ -121,8 +202,19 @@ def georgian_day(date): def year(date): - """ Returns the year - Accepted format: %m/%d/%Y + """ Returns the year. + + :param date: + The string date with this format %m/%d/%Y + :type date: + String + + :returns: + int + + :example: + >>> year('05/1/2015') + 2015 """ try: fmt = '%m/%d/%Y' @@ -132,8 +224,23 @@ def year(date): def reformat_date(date, new_fmt='%Y-%m-%d'): - """ Return reformated date. Example: 01/28/2014 & %d/%m/%Y -> 28/01/2014 - Accepted date format: %m/%d/%Y + """ Returns reformated date. + + :param date: + The string date with this format %m/%d/%Y + :type date: + String + :param new_fmt: + date format string. Default is '%Y-%m-%d' + :type date: + String + + :returns: + int + + :example: + >>> reformat_date('05/1/2015', '%d/%m/%Y') + '1/05/2015' """ try: if isinstance(date, datetime): @@ -146,7 +253,21 @@ def reformat_date(date, new_fmt='%Y-%m-%d'): def convert_to_integer_list(value): - """ convert a comma separate string to a list where all values are integers """ + """ Converts a comma separate string to a list + + :param value: + the format must be 003,003,004,004 (commas with no space) + :type value: + String + + :returns: + List + + :example: + >>> convert_to_integer_list('003,003,004,004') + ['003', '003', '004', '004'] + + """ if value and isinstance(value, str): if ',' in value: diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 3442da1..0000000 --- a/tox.ini +++ /dev/null @@ -1,7 +0,0 @@ -[tox] -envlist = py27 - -[testenv] -deps = - -r{toxinidir}/requirements/dev.txt -commands = nosetests