diff --git a/.travis.yml b/.travis.yml index 63a97aa..58cbb1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,45 @@ language: python + +sudo: false + +cache: + directories: + - ~/.cache/pip + +env: + global: + - PIP_WHEEL_DIR=$HOME/.cache/pip/wheels + - PIP_FIND_LINKS=file://$HOME/.cache/pip/wheels + +virtualenv: + system_site_packages: true + +addons: + apt: + packages: + - libgdal1h + - gdal-bin + - libgdal-dev + - libatlas-dev + - libatlas-base-dev + - gfortran + - python-numpy + - python-scipy + python: - '2.7' + before_install: -- sudo add-apt-repository -y ppa:ubuntugis/ppa -- sudo apt-get update -qq -- sudo apt-get install libgdal1h gdal-bin libgdal-dev libcurl4-gnutls-dev -- wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh -- chmod +x miniconda.sh -- "./miniconda.sh -b" -- export PATH=/home/travis/miniconda/bin:$PATH -- conda update --yes conda -- sudo rm -rf /dev/shm -- sudo ln -s /run/shm /dev/shm + - pip install -U pip + - pip install wheel + install: -- conda install --yes python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib scikit-image - six nose dateutil -- conda install --yes -c https://conda.binstar.org/osgeo gdal -- pip install --install-option="--no-cython-compile" cython -- pip uninstall requests --yes -- pip install requests==2.5.3 -- pip install -r requirements/travis.txt + - "pip wheel -r requirements/dev.txt" + - "pip install -r requirements/dev.txt" + script: - nosetests + deploy: provider: pypi user: devseed @@ -31,3 +49,12 @@ deploy: tags: true repo: developmentseed/landsat-util branch: master + +after_deploy: + if [ "$TRAVIS_BRANCH" == "master" ]; then + echo "Start Docker Hub Push" + VER=$(python -c "import landsat; print landsat.__version__") + docker build . -t developmentseed/landsat-util:$VER + docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} + docker push developmentseed/landsat-util:$VER + fi diff --git a/CHANGES.txt b/CHANGES.txt index 095a540..c1ee221 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,14 @@ Changes ======= +0.8.0 (2015-09-22) +------------------ +- Improved docs +- Add `--ndvi` flag +- Handle downloading new bands (10, 11, QA) +- Improved color correction +- Remove noise in pansharpened image processing + 0.7.0 (2015-05-29) ------------------ - New documentation diff --git a/README.rst b/README.rst index 1578ac6..2b8c418 100644 --- a/README.rst +++ b/README.rst @@ -25,6 +25,7 @@ For full documentation visit: http://landsat-util.readthedocs.org/ To run the documentation locally:: + $ pip install -r requirements/dev.txt $ cd docs $ make html @@ -39,5 +40,3 @@ Change Log +++++++++ See `CHANGES.txt `_. - - diff --git a/docs/installation.rst b/docs/installation.rst index 3a4ff23..62e00f0 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -8,7 +8,7 @@ Mac OSX $: pip install landsat-util -Ubuntu 14.10 +Ubuntu 14.04 ++++++++++++ Use pip to install landsat-util. If you are not using virtualenv, you might have to run ``pip`` as ``sudo``:: diff --git a/docs/notes.rst b/docs/notes.rst index cabe864..8aea670 100644 --- a/docs/notes.rst +++ b/docs/notes.rst @@ -8,3 +8,5 @@ Important Notes - Image processing is a very heavy and resource consuming task. Each process takes about 5-10 mins. We recommend that you run the processes in smaller badges. Pansharpening, while increasing image resolution 2x, substantially increases processing time. - Landsat-util requires at least 2GB of Memory (RAM). + +- Make sure to read over the `section on returned products `_ as it is different depending on scene acquisition date. diff --git a/docs/overview.rst b/docs/overview.rst index d0ce79f..d79e31a 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -80,3 +80,8 @@ Specify bands 3, 5 and 1:: Process *and* pansharpen a downloaded image:: $: landsat process path/to/LC80090452014008LGN00.tar.bz --pansharpen + +A note on returned products +++++++++++++++++ + +Scenes acquired after 2015 will be downloaded from `AWS Public Data Sets `_ while scenes acquired before 2015 will be downloaded from `Google Earth Engine `_. AWS provides the bands separately and so landsat-util will also pass along the bands individually if requested. In the case of Google Earth Engine, only the full, compressed image bundle is available (including all bands and metadata) and will be downloaded no matter what bands are requested. diff --git a/landsat/__init__.py b/landsat/__init__.py index a71c5c7..32a90a3 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -1 +1 @@ -__version__ = '0.7.0' +__version__ = '0.8.0' diff --git a/landsat/decorators.py b/landsat/decorators.py new file mode 100644 index 0000000..4f439f2 --- /dev/null +++ b/landsat/decorators.py @@ -0,0 +1,12 @@ +import warnings +import rasterio + + +def rasterio_decorator(func): + def wrapped_f(*args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with rasterio.drivers(): + return func(*args, **kwargs) + + return wrapped_f diff --git a/landsat/image.py b/landsat/image.py index 0a7d928..3c54526 100644 --- a/landsat/image.py +++ b/landsat/image.py @@ -2,8 +2,7 @@ # Landsat Util # License: CC0 1.0 Universal -import warnings -import sys +import os from os.path import join, isdir import tarfile import glob @@ -17,9 +16,9 @@ from skimage.util import img_as_ubyte from skimage.exposure import rescale_intensity -import settings from mixins import VerbosityMixin -from utils import get_file, timer, check_create_folder, exit +from utils import get_file, check_create_folder, exit +from decorators import rasterio_decorator class FileDoesNotExist(Exception): @@ -27,7 +26,7 @@ class FileDoesNotExist(Exception): pass -class Process(VerbosityMixin): +class BaseProcess(VerbosityMixin): """ Image procssing class @@ -66,8 +65,8 @@ def __init__(self, path, bands=None, dst_path=None, verbose=False, force_unzip=F # Landsat source path self.src_path = path.replace(get_file(path), '') - # Build destination folder if doesn't exits - self.dst_path = dst_path if dst_path else settings.PROCESSED_IMAGE + # Build destination folder if doesn't exist + self.dst_path = dst_path if dst_path else os.getcwd() self.dst_path = check_create_folder(join(self.dst_path, self.scene)) self.verbose = verbose @@ -81,179 +80,7 @@ def __init__(self, path, bands=None, dst_path=None, verbose=False, force_unzip=F for band in self.bands: self.bands_path.append(join(self.scene_path, self._get_full_filename(band))) - def run(self, pansharpen=True): - """ Executes the image processing. - - :param pansharpen: - Whether the process should also run pansharpenning. Default is True - :type pansharpen: - boolean - - :returns: - (String) the path to the processed image - """ - - self.output("* Image processing started for bands %s" % "-".join(map(str, self.bands)), normal=True) - - # Read cloud coverage from mtl file - cloud_cover = 0 - try: - with open(self.scene_path + '/' + self.scene + '_MTL.txt', 'rU') as mtl: - lines = mtl.readlines() - for line in lines: - if 'CLOUD_COVER' in line: - cloud_cover = float(line.replace('CLOUD_COVER = ', '')) - break - except IOError: - pass - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - with rasterio.drivers(): - bands = [] - - # Add band 8 for pansharpenning - if pansharpen: - self.bands.append(8) - - bands_path = [] - - for band in self.bands: - bands_path.append(join(self.scene_path, self._get_full_filename(band))) - - try: - for i, band in enumerate(self.bands): - bands.append(self._read_band(bands_path[i])) - except IOError as e: - exit(e.message, 1) - - src = rasterio.open(bands_path[-1]) - - # Get pixel size from source - self.pixel = src.affine[0] - - # Only collect src data that is needed and delete the rest - src_data = { - 'transform': src.transform, - 'crs': src.crs, - 'affine': src.affine, - 'shape': src.shape - } - del src - - crn = self._get_boundaries(src_data) - - dst_shape = src_data['shape'] - dst_corner_ys = [crn[k]['y'][1][0] for k in crn.keys()] - dst_corner_xs = [crn[k]['x'][1][0] for k in crn.keys()] - y_pixel = abs(max(dst_corner_ys) - min(dst_corner_ys)) / dst_shape[0] - x_pixel = abs(max(dst_corner_xs) - min(dst_corner_xs)) / dst_shape[1] - - dst_transform = (min(dst_corner_xs), - x_pixel, - 0.0, - max(dst_corner_ys), - 0.0, - -y_pixel) - # Delete crn since no longer needed - del crn - - new_bands = [] - for i in range(0, 3): - new_bands.append(numpy.empty(dst_shape, dtype=numpy.uint16)) - - if pansharpen: - bands[:3] = self._rescale(bands[:3]) - new_bands.append(numpy.empty(dst_shape, dtype=numpy.uint16)) - - self.output("Projecting", normal=True, arrow=True) - for i, band in enumerate(bands): - self.output("band %s" % self.bands[i], normal=True, color='green', indent=1) - reproject(band, new_bands[i], src_transform=src_data['transform'], src_crs=src_data['crs'], - dst_transform=dst_transform, dst_crs=self.dst_crs, resampling=RESAMPLING.nearest) - - # Bands are no longer needed - del bands - - if pansharpen: - new_bands = self._pansharpenning(new_bands) - del self.bands[3] - - self.output("Final Steps", normal=True, arrow=True) - - output_file = '%s_bands_%s' % (self.scene, "".join(map(str, self.bands))) - - if pansharpen: - output_file += '_pan' - - output_file += '.TIF' - output_file = join(self.dst_path, output_file) - - output = rasterio.open(output_file, 'w', driver='GTiff', - width=dst_shape[1], height=dst_shape[0], - count=3, dtype=numpy.uint8, - nodata=0, transform=dst_transform, photometric='RGB', - crs=self.dst_crs) - - for i, band in enumerate(new_bands): - # Color Correction - band = self._color_correction(band, self.bands[i], 0, cloud_cover) - - output.write_band(i+1, img_as_ubyte(band)) - - new_bands[i] = None - self.output("Writing to file", normal=True, color='green', indent=1) - return output_file - - def _pansharpenning(self, bands): - - self.output("Pansharpening", normal=True, arrow=True) - # Pan sharpening - m = sum(bands[:3]) - m = m + 0.1 - - self.output("calculating pan ratio", normal=True, color='green', indent=1) - pan = 1/m * bands[-1] - - del m - del bands[3] - self.output("computing bands", normal=True, color='green', indent=1) - - for i, band in enumerate(bands): - bands[i] = band * pan - - del pan - - return bands - - def _color_correction(self, band, band_id, low, cloud_cover): - band = band.astype(numpy.uint16) - - self.output("Color correcting band %s" % band_id, normal=True, color='green', indent=1) - p_low, cloud_cut_low = self._percent_cut(band, low, 100 - (cloud_cover * 3 / 4)) - temp = numpy.zeros(numpy.shape(band), dtype=numpy.uint16) - cloud_divide = 65000 - cloud_cover * 100 - mask = numpy.logical_and(band < cloud_cut_low, band > 0) - temp[mask] = rescale_intensity(band[mask], in_range=(p_low, cloud_cut_low), out_range=(256, cloud_divide)) - temp[band >= cloud_cut_low] = rescale_intensity(band[band >= cloud_cut_low], out_range=(cloud_divide, 65535)) - return temp - - def _read_band(self, band_path): - """ Reads a band with rasterio """ - return rasterio.open(band_path).read_band(1) - - def _rescale(self, bands): - """ Rescale bands """ - self.output("Rescaling", normal=True, arrow=True) - - for key, band in enumerate(bands): - self.output("band %s" % self.bands[key], normal=True, color='green', indent=1) - bands[key] = sktransform.rescale(band, 2) - bands[key] = (bands[key] * 65535).astype('uint16') - - return bands - - def _get_boundaries(self, src): + def _get_boundaries(self, src, shape): self.output("Getting boundaries", normal=True, arrow=True) output = {'ul': {'x': [0, 0], 'y': [0, 0]}, # ul: upper left @@ -286,10 +113,31 @@ def _get_boundaries(self, src): [output['lr']['x'][0]], [output['lr']['y'][0]]) - return output + dst_corner_ys = [output[k]['y'][1][0] for k in output.keys()] + dst_corner_xs = [output[k]['x'][1][0] for k in output.keys()] + y_pixel = abs(max(dst_corner_ys) - min(dst_corner_ys)) / shape[0] + x_pixel = abs(max(dst_corner_xs) - min(dst_corner_xs)) / shape[1] - def _percent_cut(self, color, low, high): - return numpy.percentile(color[numpy.logical_and(color > 0, color < 65535)], (low, high)) + return (min(dst_corner_xs), x_pixel, 0.0, max(dst_corner_ys), 0.0, -y_pixel) + + def _read_bands(self): + """ Reads a band with rasterio """ + bands = [] + + try: + for i, band in enumerate(self.bands): + bands.append(rasterio.open(self.bands_path[i]).read_band(1)) + except IOError as e: + exit(e.message, 1) + + return bands + + def _warp(self, proj_data, bands, new_bands): + self.output("Projecting", normal=True, arrow=True) + for i, band in enumerate(bands): + self.output("band %s" % self.bands[i], normal=True, color='green', indent=1) + reproject(band, new_bands[i], src_transform=proj_data['transform'], src_crs=proj_data['crs'], + dst_transform=proj_data['dst_transform'], dst_crs=self.dst_crs, resampling=RESAMPLING.nearest) def _unzip(self, src, dst, scene, force_unzip=False): """ Unzip tar files """ @@ -326,10 +174,200 @@ def _check_if_zipped(self, path): return False + def _read_cloud_cover(self): + try: + with open(self.scene_path + '/' + self.scene + '_MTL.txt', 'rU') as mtl: + lines = mtl.readlines() + for line in lines: + if 'CLOUD_COVER' in line: + return float(line.replace('CLOUD_COVER = ', '')) + except IOError: + return 0 + + def _get_image_data(self): + src = rasterio.open(self.bands_path[-1]) + + # Get pixel size from source + self.pixel = src.affine[0] + + # Only collect src data that is needed and delete the rest + image_data = { + 'transform': src.transform, + 'crs': src.crs, + 'affine': src.affine, + 'shape': src.shape, + 'dst_transform': None + } + + image_data['dst_transform'] = self._get_boundaries(image_data, image_data['shape']) + + return image_data + + def _generate_new_bands(self, shape): + new_bands = [] + for i in range(0, 3): + new_bands.append(numpy.empty(shape, dtype=numpy.uint16)) + + return new_bands + + @rasterio_decorator + def _write_to_file(self, new_bands, suffix=None, **kwargs): + + # Read cloud coverage from mtl file + cloud_cover = self._read_cloud_cover() + + self.output("Final Steps", normal=True, arrow=True) + + output_file = '%s_bands_%s' % (self.scene, "".join(map(str, self.bands))) + + if suffix: + output_file += suffix + + output_file += '.TIF' + output_file = join(self.dst_path, output_file) + + output = rasterio.open(output_file, 'w', **kwargs) + + for i, band in enumerate(new_bands): + # Color Correction + band = self._color_correction(band, self.bands[i], 0, cloud_cover) + + output.write_band(i+1, img_as_ubyte(band)) + + new_bands[i] = None + self.output("Writing to file", normal=True, color='green', indent=1) + + return output_file + + def _color_correction(self, band, band_id, low, cloud_cover): + band = band.astype(numpy.uint16) + + self.output("Color correcting band %s" % band_id, normal=True, color='green', indent=1) + p_low, cloud_cut_low = self._percent_cut(band, low, 100 - (cloud_cover * 3 / 4)) + temp = numpy.zeros(numpy.shape(band), dtype=numpy.uint16) + cloud_divide = 65000 - cloud_cover * 100 + mask = numpy.logical_and(band < cloud_cut_low, band > 0) + temp[mask] = rescale_intensity(band[mask], in_range=(p_low, cloud_cut_low), out_range=(256, cloud_divide)) + temp[band >= cloud_cut_low] = rescale_intensity(band[band >= cloud_cut_low], out_range=(cloud_divide, 65535)) + return temp + + def _percent_cut(self, color, low, high): + return numpy.percentile(color[numpy.logical_and(color > 0, color < 65535)], (low, high)) + + +class Simple(BaseProcess): + + @rasterio_decorator + def run(self): + """ Executes the image processing. + + :returns: + (String) the path to the processed image + """ + + self.output("* Image processing started for bands %s" % "-".join(map(str, self.bands)), normal=True) + + bands = self._read_bands() + image_data = self._get_image_data() + + new_bands = self._generate_new_bands(image_data['shape']) -if __name__ == '__main__': + self._warp(image_data, bands, new_bands) - with timer(): - p = Process(sys.argv[1]) + # Bands are no longer needed + del bands - print p.run(sys.argv[2] == 't') + rasterio_options = { + 'driver': 'GTiff', + 'width': image_data['shape'][1], + 'height': image_data['shape'][0], + 'count': 3, + 'dtype': numpy.uint8, + 'nodata': 0, + 'transform': image_data['dst_transform'], + 'photometric': 'RGB', + 'crs': self.dst_crs + } + + return self._write_to_file(new_bands, **rasterio_options) + + +class PanSharpen(BaseProcess): + + def __init__(self, path, bands=None, dst_path=None, verbose=False, force_unzip=False): + if bands: + bands.append(8) + else: + bands = [4, 3, 2, 8] + super(PanSharpen, self).__init__(path, bands, dst_path, verbose, force_unzip) + + @rasterio_decorator + def run(self): + """ Executes the pansharpen image processing. + :returns: + (String) the path to the processed image + """ + + self.output("* PanSharpened Image processing started for bands %s" % "-".join(map(str, self.bands)), normal=True) + + bands = self._read_bands() + image_data = self._get_image_data() + + new_bands = self._generate_new_bands(image_data['shape']) + + bands[:3] = self._rescale(bands[:3]) + new_bands.append(numpy.empty(image_data['shape'], dtype=numpy.uint16)) + + self._warp(image_data, bands, new_bands) + + # Bands are no longer needed + del bands + + new_bands = self._pansharpenning(new_bands) + del self.bands[3] + + rasterio_options = { + 'driver': 'GTiff', + 'width': image_data['shape'][1], + 'height': image_data['shape'][0], + 'count': 3, + 'dtype': numpy.uint8, + 'nodata': 0, + 'transform': image_data['dst_transform'], + 'photometric': 'RGB', + 'crs': self.dst_crs + } + + return self._write_to_file(new_bands, '_pan', **rasterio_options) + + def _pansharpenning(self, bands): + + self.output("Pansharpening", normal=True, arrow=True) + # Pan sharpening + m = sum(bands[:3]) + m = m + 0.1 + + self.output("calculating pan ratio", normal=True, color='green', indent=1) + pan = 1/m * bands[-1] + + del m + del bands[3] + self.output("computing bands", normal=True, color='green', indent=1) + + for i, band in enumerate(bands): + bands[i] = band * pan + + del pan + + return bands + + def _rescale(self, bands): + """ Rescale bands """ + self.output("Rescaling", normal=True, arrow=True) + + for key, band in enumerate(bands): + self.output("band %s" % self.bands[key], normal=True, color='green', indent=1) + bands[key] = sktransform.rescale(band, 2) + bands[key] = (bands[key] * 65535).astype('uint16') + + return bands diff --git a/landsat/landsat.py b/landsat/landsat.py index bb034df..3a6a152 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -18,7 +18,8 @@ from uploader import Uploader from utils import reformat_date, convert_to_integer_list, timer, exit, get_file from mixins import VerbosityMixin -from image import Process, FileDoesNotExist +from image import Simple, PanSharpen, FileDoesNotExist +from ndvi import NDVIWithManualColorMap, NDVI from __init__ import __version__ import settings @@ -74,6 +75,8 @@ --pansharpen Whether to also pansharpen the processed image. Pansharpening requires larger memory + --ndvi Whether to run the NDVI process. If used, bands parameter is disregarded + -u --upload Upload to S3 after the image processing completed --key Amazon S3 Access Key (You can also be set AWS_ACCESS_KEY_ID as @@ -102,6 +105,8 @@ --pansharpen Whether to also pansharpen the process image. Pansharpening requires larger memory + --ndvi Whether to run the NDVI process. If used, bands parameter is disregarded + -v, --verbose Show verbose output -h, --help Show this help message and exit @@ -175,6 +180,8 @@ def args_options(): parser_download.add_argument('--pansharpen', action='store_true', help='Whether to also pansharpen the process ' 'image. Pansharpening requires larger memory') + parser_download.add_argument('--ndvi', action='store_true', + help='Whether to run the NDVI process. If used, bands parameter is disregarded') parser_download.add_argument('-u', '--upload', action='store_true', help='Upload to S3 after the image processing completed') parser_download.add_argument('--key', help='Amazon S3 Access Key (You can also be set AWS_ACCESS_KEY_ID as ' @@ -191,6 +198,10 @@ def args_options(): parser_process.add_argument('--pansharpen', action='store_true', help='Whether to also pansharpen the process ' 'image. Pansharpening requires larger memory') + parser_process.add_argument('--ndvi', action='store_true', + help='Whether to run the NDVI process. If used, bands parameter is disregarded') + parser_process.add_argument('--ndvi1', action='store_true', + help='Whether to run the NDVI process. If used, bands parameter is disregarded') parser_process.add_argument('-b', '--bands', help='specify band combinations. Default is 432' 'Example: --bands 321') parser_process.add_argument('-v', '--verbose', action='store_true', @@ -231,7 +242,7 @@ def main(args): if args.subs == 'process': verbose = True if args.verbose else False force_unzip = True if args.force_unzip else False - stored = process_image(args.path, args.bands, verbose, args.pansharpen, force_unzip) + stored = process_image(args.path, args.bands, verbose, args.pansharpen, args.ndvi, force_unzip, args.ndvi1) if args.upload: u = Uploader(args.key, args.secret, args.region) @@ -280,6 +291,8 @@ def main(args): bands = convert_to_integer_list(args.bands) if args.pansharpen: bands.append(8) + if args.ndvi: + bands = [4, 5] downloaded = d.download(args.scenes, bands) @@ -295,7 +308,7 @@ def main(args): if src == 'google': path = path + '.tar.bz' - stored = process_image(path, args.bands, False, args.pansharpen, force_unzip) + stored = process_image(path, args.bands, False, args.pansharpen, args.ndvi, force_unzip) if args.upload: try: @@ -315,7 +328,7 @@ def main(args): return ['The SceneID provided was incorrect', 1] -def process_image(path, bands=None, verbose=False, pansharpen=False, force_unzip=None): +def process_image(path, bands=None, verbose=False, pansharpen=False, ndvi=False, force_unzip=None, ndvi1=False): """ Handles constructing and image process. :param path: @@ -340,13 +353,22 @@ def process_image(path, bands=None, verbose=False, pansharpen=False, force_unzip """ try: bands = convert_to_integer_list(bands) - p = Process(path, bands=bands, verbose=verbose, force_unzip=force_unzip) + if pansharpen: + p = PanSharpen(path, bands=bands, dst_path=settings.PROCESSED_IMAGE, + verbose=verbose, force_unzip=force_unzip) + elif ndvi1: + p = NDVI(path, verbose=verbose, dst_path=settings.PROCESSED_IMAGE, force_unzip=force_unzip) + elif ndvi: + p = NDVIWithManualColorMap(path, dst_path=settings.PROCESSED_IMAGE, + verbose=verbose, force_unzip=force_unzip) + else: + p = Simple(path, bands=bands, dst_path=settings.PROCESSED_IMAGE, verbose=verbose, force_unzip=force_unzip) except IOError: exit("Zip file corrupted", 1) except FileDoesNotExist as e: exit(e.message, 1) - return p.run(pansharpen) + return p.run() def __main__(): diff --git a/landsat/ndvi.py b/landsat/ndvi.py new file mode 100644 index 0000000..d3f112d --- /dev/null +++ b/landsat/ndvi.py @@ -0,0 +1,354 @@ +from os.path import join + +import rasterio +import numpy + +from decorators import rasterio_decorator +from image import BaseProcess + + +class NDVI(BaseProcess): + + def __init__(self, path, bands=None, dst_path=None, verbose=False, force_unzip=False): + bands = [4, 5] + self.cmap = {0: (255, 255, 255, 0), + 1: (250, 250, 250, 255), + 2: (246, 246, 246, 255), + 3: (242, 242, 242, 255), + 4: (238, 238, 238, 255), + 5: (233, 233, 233, 255), + 6: (229, 229, 229, 255), + 7: (225, 225, 225, 255), + 8: (221, 221, 221, 255), + 9: (216, 216, 216, 255), + 10: (212, 212, 212, 255), + 11: (208, 208, 208, 255), + 12: (204, 204, 204, 255), + 13: (200, 200, 200, 255), + 14: (195, 195, 195, 255), + 15: (191, 191, 191, 255), + 16: (187, 187, 187, 255), + 17: (183, 183, 183, 255), + 18: (178, 178, 178, 255), + 19: (174, 174, 174, 255), + 20: (170, 170, 170, 255), + 21: (166, 166, 166, 255), + 22: (161, 161, 161, 255), + 23: (157, 157, 157, 255), + 24: (153, 153, 153, 255), + 25: (149, 149, 149, 255), + 26: (145, 145, 145, 255), + 27: (140, 140, 140, 255), + 28: (136, 136, 136, 255), + 29: (132, 132, 132, 255), + 30: (128, 128, 128, 255), + 31: (123, 123, 123, 255), + 32: (119, 119, 119, 255), + 33: (115, 115, 115, 255), + 34: (111, 111, 111, 255), + 35: (106, 106, 106, 255), + 36: (102, 102, 102, 255), + 37: (98, 98, 98, 255), + 38: (94, 94, 94, 255), + 39: (90, 90, 90, 255), + 40: (85, 85, 85, 255), + 41: (81, 81, 81, 255), + 42: (77, 77, 77, 255), + 43: (73, 73, 73, 255), + 44: (68, 68, 68, 255), + 45: (64, 64, 64, 255), + 46: (60, 60, 60, 255), + 47: (56, 56, 56, 255), + 48: (52, 52, 52, 255), + 49: (56, 56, 56, 255), + 50: (60, 60, 60, 255), + 51: (64, 64, 64, 255), + 52: (68, 68, 68, 255), + 53: (73, 73, 73, 255), + 54: (77, 77, 77, 255), + 55: (81, 81, 81, 255), + 56: (85, 85, 85, 255), + 57: (90, 90, 90, 255), + 58: (94, 94, 94, 255), + 59: (98, 98, 98, 255), + 60: (102, 102, 102, 255), + 61: (106, 106, 106, 255), + 62: (111, 111, 111, 255), + 63: (115, 115, 115, 255), + 64: (119, 119, 119, 255), + 65: (123, 123, 123, 255), + 66: (128, 128, 128, 255), + 67: (132, 132, 132, 255), + 68: (136, 136, 136, 255), + 69: (140, 140, 140, 255), + 70: (145, 145, 145, 255), + 71: (149, 149, 149, 255), + 72: (153, 153, 153, 255), + 73: (157, 157, 157, 255), + 74: (161, 161, 161, 255), + 75: (166, 166, 166, 255), + 76: (170, 170, 170, 255), + 77: (174, 174, 174, 255), + 78: (178, 178, 178, 255), + 79: (183, 183, 183, 255), + 80: (187, 187, 187, 255), + 81: (191, 191, 191, 255), + 82: (195, 195, 195, 255), + 83: (200, 200, 200, 255), + 84: (204, 204, 204, 255), + 85: (208, 208, 208, 255), + 86: (212, 212, 212, 255), + 87: (216, 216, 216, 255), + 88: (221, 221, 221, 255), + 89: (225, 225, 225, 255), + 90: (229, 229, 229, 255), + 91: (233, 233, 233, 255), + 92: (238, 238, 238, 255), + 93: (242, 242, 242, 255), + 94: (246, 246, 246, 255), + 95: (250, 250, 250, 255), + 96: (255, 255, 255, 255), + 97: (250, 250, 250, 255), + 98: (245, 245, 245, 255), + 99: (240, 240, 240, 255), + 100: (235, 235, 235, 255), + 101: (230, 230, 230, 255), + 102: (225, 225, 225, 255), + 103: (220, 220, 220, 255), + 104: (215, 215, 215, 255), + 105: (210, 210, 210, 255), + 106: (205, 205, 205, 255), + 107: (200, 200, 200, 255), + 108: (195, 195, 195, 255), + 109: (190, 190, 190, 255), + 110: (185, 185, 185, 255), + 111: (180, 180, 180, 255), + 112: (175, 175, 175, 255), + 113: (170, 170, 170, 255), + 114: (165, 165, 165, 255), + 115: (160, 160, 160, 255), + 116: (155, 155, 155, 255), + 117: (151, 151, 151, 255), + 118: (146, 146, 146, 255), + 119: (141, 141, 141, 255), + 120: (136, 136, 136, 255), + 121: (131, 131, 131, 255), + 122: (126, 126, 126, 255), + 123: (121, 121, 121, 255), + 124: (116, 116, 116, 255), + 125: (111, 111, 111, 255), + 126: (106, 106, 106, 255), + 127: (101, 101, 101, 255), + 128: (96, 96, 96, 255), + 129: (91, 91, 91, 255), + 130: (86, 86, 86, 255), + 131: (81, 81, 81, 255), + 132: (76, 76, 76, 255), + 133: (71, 71, 71, 255), + 134: (66, 66, 66, 255), + 135: (61, 61, 61, 255), + 136: (56, 56, 56, 255), + 137: (66, 66, 80, 255), + 138: (77, 77, 105, 255), + 139: (87, 87, 130, 255), + 140: (98, 98, 155, 255), + 141: (108, 108, 180, 255), + 142: (119, 119, 205, 255), + 143: (129, 129, 230, 255), + 144: (140, 140, 255, 255), + 145: (131, 147, 239, 255), + 146: (122, 154, 223, 255), + 147: (113, 161, 207, 255), + 148: (105, 168, 191, 255), + 149: (96, 175, 175, 255), + 150: (87, 183, 159, 255), + 151: (78, 190, 143, 255), + 152: (70, 197, 127, 255), + 153: (61, 204, 111, 255), + 154: (52, 211, 95, 255), + 155: (43, 219, 79, 255), + 156: (35, 226, 63, 255), + 157: (26, 233, 47, 255), + 158: (17, 240, 31, 255), + 159: (8, 247, 15, 255), + 160: (0, 255, 0, 255), + 161: (7, 255, 0, 255), + 162: (15, 255, 0, 255), + 163: (23, 255, 0, 255), + 164: (31, 255, 0, 255), + 165: (39, 255, 0, 255), + 166: (47, 255, 0, 255), + 167: (55, 255, 0, 255), + 168: (63, 255, 0, 255), + 169: (71, 255, 0, 255), + 170: (79, 255, 0, 255), + 171: (87, 255, 0, 255), + 172: (95, 255, 0, 255), + 173: (103, 255, 0, 255), + 174: (111, 255, 0, 255), + 175: (119, 255, 0, 255), + 176: (127, 255, 0, 255), + 177: (135, 255, 0, 255), + 178: (143, 255, 0, 255), + 179: (151, 255, 0, 255), + 180: (159, 255, 0, 255), + 181: (167, 255, 0, 255), + 182: (175, 255, 0, 255), + 183: (183, 255, 0, 255), + 184: (191, 255, 0, 255), + 185: (199, 255, 0, 255), + 186: (207, 255, 0, 255), + 187: (215, 255, 0, 255), + 188: (223, 255, 0, 255), + 189: (231, 255, 0, 255), + 190: (239, 255, 0, 255), + 191: (247, 255, 0, 255), + 192: (255, 255, 0, 255), + 193: (255, 249, 0, 255), + 194: (255, 244, 0, 255), + 195: (255, 239, 0, 255), + 196: (255, 233, 0, 255), + 197: (255, 228, 0, 255), + 198: (255, 223, 0, 255), + 199: (255, 217, 0, 255), + 200: (255, 212, 0, 255), + 201: (255, 207, 0, 255), + 202: (255, 201, 0, 255), + 203: (255, 196, 0, 255), + 204: (255, 191, 0, 255), + 205: (255, 185, 0, 255), + 206: (255, 180, 0, 255), + 207: (255, 175, 0, 255), + 208: (255, 170, 0, 255), + 209: (255, 164, 0, 255), + 210: (255, 159, 0, 255), + 211: (255, 154, 0, 255), + 212: (255, 148, 0, 255), + 213: (255, 143, 0, 255), + 214: (255, 138, 0, 255), + 215: (255, 132, 0, 255), + 216: (255, 127, 0, 255), + 217: (255, 122, 0, 255), + 218: (255, 116, 0, 255), + 219: (255, 111, 0, 255), + 220: (255, 106, 0, 255), + 221: (255, 100, 0, 255), + 222: (255, 95, 0, 255), + 223: (255, 90, 0, 255), + 224: (255, 85, 0, 255), + 225: (255, 79, 0, 255), + 226: (255, 74, 0, 255), + 227: (255, 69, 0, 255), + 228: (255, 63, 0, 255), + 229: (255, 58, 0, 255), + 230: (255, 53, 0, 255), + 231: (255, 47, 0, 255), + 232: (255, 42, 0, 255), + 233: (255, 37, 0, 255), + 234: (255, 31, 0, 255), + 235: (255, 26, 0, 255), + 236: (255, 21, 0, 255), + 237: (255, 15, 0, 255), + 238: (255, 10, 0, 255), + 239: (255, 5, 0, 255), + 240: (255, 0, 0, 255), + 241: (255, 0, 15, 255), + 242: (255, 0, 31, 255), + 243: (255, 0, 47, 255), + 244: (255, 0, 63, 255), + 245: (255, 0, 79, 255), + 246: (255, 0, 95, 255), + 247: (255, 0, 111, 255), + 248: (255, 0, 127, 255), + 249: (255, 0, 143, 255), + 250: (255, 0, 159, 255), + 251: (255, 0, 175, 255), + 252: (255, 0, 191, 255), + 253: (255, 0, 207, 255), + 254: (255, 0, 223, 255), + 255: (255, 0, 239, 255)} + super(NDVI, self).__init__(path, bands, dst_path, verbose, force_unzip) + + @rasterio_decorator + def run(self): + """ + Executes NDVI processing + """ + self.output("* NDVI processing started.", normal=True) + + bands = self._read_bands() + image_data = self._get_image_data() + + new_bands = [] + for i in range(0, 2): + new_bands.append(numpy.empty(image_data['shape'], dtype=numpy.float32)) + + self._warp(image_data, bands, new_bands) + + # Bands are no longer needed + del bands + + calc_band = numpy.true_divide((new_bands[1] - new_bands[0]), (new_bands[1] + new_bands[0])) + + output_band = numpy.rint((calc_band + 1) * 255 / 2).astype(numpy.uint8) + + output_file = '%s_NDVI.TIF' % (self.scene) + output_file = join(self.dst_path, output_file) + + return self.write_band(output_band, output_file, image_data) + + def write_band(self, output_band, output_file, image_data): + + # from http://publiclab.org/notes/cfastie/08-26-2014/new-ndvi-colormap + with rasterio.open(output_file, 'w', driver='GTiff', + width=image_data['shape'][1], + height=image_data['shape'][0], + count=1, + dtype=numpy.uint8, + nodata=0, + transform=image_data['dst_transform'], + crs=self.dst_crs) as output: + + output.write_band(1, output_band) + + cmap = {k: v[:3] for k, v in self.cmap.iteritems()} + output.write_colormap(1, cmap) + self.output("Writing to file", normal=True, color='green', indent=1) + return output_file + + +class NDVIWithManualColorMap(NDVI): + + def manual_colormap(self, n, i): + return self.cmap[n][i] + + def write_band(self, output_band, output_file, image_data): + # colormaps will overwrite our transparency masks so we will manually + # create three RGB bands + + self.output("Creating Manual ColorMap", normal=True, arrow=True) + self.cmap[0] = (0, 0, 0, 255) + + v_manual_colormap = numpy.vectorize(self.manual_colormap, otypes=[numpy.uint8]) + rgb_bands = [] + for i in range(3): + rgb_bands.append(v_manual_colormap(output_band, i)) + + with rasterio.drivers(GDAL_TIFF_INTERNAL_MASK=True): + with rasterio.open(output_file, 'w', driver='GTiff', + width=image_data['shape'][1], + height=image_data['shape'][0], + count=3, + dtype=numpy.uint8, + nodata=0, + photometric='RGB', + transform=image_data['dst_transform'], + crs=self.dst_crs) as output: + + for i in range(3): + output.write_band(i+1, rgb_bands[i]) + # output.write_colormap(1, cmap) + # output.write_mask(no_data_mask) + + self.output("Writing to file", normal=True, color='green', indent=1) + return output_file diff --git a/landsat/utils.py b/landsat/utils.py index b5dc6e3..3002cb8 100644 --- a/landsat/utils.py +++ b/landsat/utils.py @@ -268,13 +268,13 @@ def convert_to_integer_list(value): ['003', '003', '004', '004'] """ - - if value and isinstance(value, str): - if ',' in value: - value = re.sub('[^0-9,]', '', value) - new_list = value.split(',') - else: - new_list = re.findall('[0-9]', value) - return new_list if new_list else None - else: + if isinstance(value, list) or value is None: return value + else: + s = re.findall('(10|11|QA|[0-9])', value) + for k, v in enumerate(s): + try: + s[k] = int(v) + except ValueError: + pass + return s diff --git a/requirements/base.txt b/requirements/base.txt index a17e798..2cdd296 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,10 +1,10 @@ -requests==2.5.3 +requests==2.7.0 python-dateutil>=2.4.2 termcolor>=1.1.0 -numpy>=1.9.2 -rasterio>=0.21.0 +numpy>=1.9.3 +rasterio>=0.26.0 six==1.9.0 +scipy>=0.16.0 scikit-image>=0.11.3 -homura>=0.1.1 -scipy>=0.15.1 +homura>=0.1.2 boto>=2.38.0 diff --git a/requirements/dev.txt b/requirements/dev.txt index 05547bf..034bd45 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,7 +1,7 @@ -r base.txt pdoc>=0.3.1 -nose>=1.3.6 -coverage>=3.7.1 +nose>=1.3.7 +coverage>=4.0 Sphinx>=1.3.1 -wheel>=0.24.0 -mock>=1.0.1 +wheel>=0.26.0 +mock>=1.3.0 diff --git a/requirements/travis.txt b/requirements/travis.txt deleted file mode 100644 index 2509138..0000000 --- a/requirements/travis.txt +++ /dev/null @@ -1,6 +0,0 @@ -rasterio>=0.21.0 -homura>=0.1.1 -termcolor>=1.1.0 -python-dateutil>=2.4.2 -mock>=1.0.1 -boto>=2.38.0 diff --git a/setup.py b/setup.py index b159bae..4b85d30 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,8 @@ def readme(): return f.read() test_requirements = [ - 'nose>=1.3.6', - 'mock>=1.0.1' + 'nose>=1.3.7', + 'mock>=1.3.0' ] setup( @@ -35,15 +35,15 @@ def readme(): license='CCO', platforms='Posix; MacOS X; Windows', install_requires=[ - 'requests==2.5.3', + 'requests==2.7.0', 'python-dateutil>=2.4.2', - 'numpy>=1.9.2', + 'numpy>=1.9.3', 'termcolor>=1.1.0', - 'rasterio>=0.21.0', + 'rasterio>=0.26.0', 'six==1.9.0', - 'scipy>=0.15.1', + 'scipy>=0.16.0', 'scikit-image>=0.11.3', - 'homura>=0.1.1', + 'homura>=0.1.2', 'boto>=2.38.0' ], test_suite='nose.collector', diff --git a/landsat/tests/__init__.py b/tests/__init__.py similarity index 100% rename from landsat/tests/__init__.py rename to tests/__init__.py diff --git a/landsat/tests/mocks.py b/tests/mocks.py similarity index 100% rename from landsat/tests/mocks.py rename to tests/mocks.py diff --git a/landsat/tests/samples/mock_upload b/tests/samples/mock_upload similarity index 100% rename from landsat/tests/samples/mock_upload rename to tests/samples/mock_upload diff --git a/landsat/tests/samples/test.tar.bz2 b/tests/samples/test.tar.bz2 similarity index 100% rename from landsat/tests/samples/test.tar.bz2 rename to tests/samples/test.tar.bz2 diff --git a/landsat/tests/test_download.py b/tests/test_download.py similarity index 93% rename from landsat/tests/test_download.py rename to tests/test_download.py index 4d7d06d..14ed7e7 100644 --- a/landsat/tests/test_download.py +++ b/tests/test_download.py @@ -4,7 +4,6 @@ """Tests for downloader""" import os -import sys import errno import shutil import unittest @@ -12,13 +11,8 @@ import mock -try: - from landsat.downloader import Downloader, RemoteFileDoesntExist, IncorrectSceneId - from landsat.settings import GOOGLE_STORAGE, S3_LANDSAT -except ImportError: - sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../landsat'))) - from landsat.downloader import Downloader, RemoteFileDoesntExist, IncorrectSceneId - from landsat.settings import GOOGLE_STORAGE, S3_LANDSAT +from landsat.downloader import Downloader, RemoteFileDoesntExist, IncorrectSceneId +from landsat.settings import GOOGLE_STORAGE, S3_LANDSAT class TestDownloader(unittest.TestCase): diff --git a/landsat/tests/test_image.py b/tests/test_image.py similarity index 52% rename from landsat/tests/test_image.py rename to tests/test_image.py index af762cd..3f87af5 100644 --- a/landsat/tests/test_image.py +++ b/tests/test_image.py @@ -4,17 +4,13 @@ """Tests for image processing""" from os.path import join, abspath, dirname, exists -import sys import errno import shutil import unittest from tempfile import mkdtemp -try: - from landsat.image import Process -except ImportError: - sys.path.append(abspath(join(dirname(__file__), '../landsat'))) - from landsat.image import Process +from landsat.image import Simple, PanSharpen +from landsat.ndvi import NDVI, NDVIWithManualColorMap class TestProcess(unittest.TestCase): @@ -24,7 +20,6 @@ def setUpClass(cls): cls.base_dir = abspath(dirname(__file__)) cls.temp_folder = mkdtemp() cls.landsat_image = join(cls.base_dir, 'samples/test.tar.bz2') - cls.p = Process(path=cls.landsat_image, dst_path=cls.temp_folder) @classmethod def tearDownClass(cls): @@ -36,25 +31,42 @@ def tearDownClass(cls): if exc.errno != errno.ENOENT: raise - def test_run(self): + def test_simple_no_bands(self): - # test with no bands - self.p.run(False) + p = Simple(path=self.landsat_image, dst_path=self.temp_folder) + p.run() self.assertTrue(exists(join(self.temp_folder, 'test', 'test_bands_432.TIF'))) - # test with bands - self.p.bands = [1, 2, 3] - self.p.run(False) + def test_simple_with_bands(self): + + p = Simple(path=self.landsat_image, bands=[1, 2, 3], dst_path=self.temp_folder) + p.run() self.assertTrue(exists(join(self.temp_folder, 'test', 'test_bands_123.TIF'))) - # test with pansharpen - self.p.bands = [4, 3, 2] - self.p.run() - print self.temp_folder - self.assertTrue(exists(join(self.temp_folder, 'test', 'test_bands_432_pan.TIF'))) + def test_simple_with_zip_file(self): + + p = Simple(path=self.landsat_image, dst_path=self.temp_folder) # test from an unzip file self.path = join(self.base_dir, 'samples', 'test') - self.p.run(False) + p.run() self.assertTrue(exists(join(self.temp_folder, 'test', 'test_bands_432.TIF'))) + def test_pansharpen(self): + # test with pansharpen + + p = PanSharpen(path=self.landsat_image, bands=[4, 3, 2], dst_path=self.temp_folder) + p.run() + self.assertTrue(exists(join(self.temp_folder, 'test', 'test_bands_432_pan.TIF'))) + + def test_ndvi(self): + + p = NDVI(path=self.landsat_image, dst_path=self.temp_folder) + print p.run() + self.assertTrue(exists(join(self.temp_folder, 'test', 'test_NDVI.TIF'))) + + def test_ndvi_with_manual_colormap(self): + + p = NDVIWithManualColorMap(path=self.landsat_image, dst_path=self.temp_folder) + print p.run() + self.assertTrue(exists(join(self.temp_folder, 'test', 'test_NDVI.TIF'))) diff --git a/landsat/tests/test_landsat.py b/tests/test_landsat.py similarity index 82% rename from landsat/tests/test_landsat.py rename to tests/test_landsat.py index 6d8d9a3..9994738 100644 --- a/landsat/tests/test_landsat.py +++ b/tests/test_landsat.py @@ -3,19 +3,14 @@ """Tests for landsat""" -import sys import unittest import subprocess import errno import shutil -from os.path import join, abspath, dirname +from os.path import join import mock -try: - import landsat.landsat as landsat -except ImportError: - sys.path.append(abspath(join(dirname(__file__), '../landsat'))) - import landsat.landsat as landsat +import landsat.landsat as landsat class TestLandsat(unittest.TestCase): @@ -80,7 +75,7 @@ def test_download_correct(self, mock_downloader): args = ['download', 'LC80010092015051LGN00', '-b', '11,', '-d', self.mock_path] output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(download_dir=self.mock_path) - mock_downloader.return_value.download.assert_called_with(['LC80010092015051LGN00'], ['11', '']) + mock_downloader.return_value.download.assert_called_with(['LC80010092015051LGN00'], [11]) self.assertEquals(output, ['Download Completed', 0]) def test_download_incorrect(self): @@ -100,24 +95,32 @@ def test_download_process_continuous(self, mock_downloader, mock_process): args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', self.mock_path, '-p'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], ['4', '3', '2']) - mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, False) + mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2]) + mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, False, False) self.assertEquals(output, ["Image Processing Completed", 0]) # Call with force unzip flag args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', self.mock_path, '-p', '--force-unzip'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], ['4', '3', '2']) - mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, True) + mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2]) + mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, False, True) self.assertEquals(output, ["Image Processing Completed", 0]) # Call with pansharpen args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', self.mock_path, '-p', '--pansharpen'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], ['4', '3', '2', 8]) - mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, True, False) + mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2, 8]) + mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, True, False, False) + self.assertEquals(output, ["Image Processing Completed", 0]) + + # Call with ndvi + args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', + self.mock_path, '-p', '--ndvi'] + output = landsat.main(self.parser.parse_args(args)) + mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 5]) + mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, True, False) self.assertEquals(output, ["Image Processing Completed", 0]) @mock.patch('landsat.landsat.Uploader') @@ -132,8 +135,8 @@ def test_download_process_continuous_with_upload(self, mock_downloader, mock_pro args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.mock_path, '-p', '-u', '--key', 'somekey', '--secret', 'somesecret', '--bucket', 'mybucket', '--region', 'this'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2']) - mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False) + mock_downloader.assert_called_with(['LC80010092015051LGN00'], [4, 3, 2]) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False, False) mock_upload.assert_called_with('somekey', 'somesecret', 'this') mock_upload.return_value.run.assert_called_with('mybucket', 'image.TIF', 'image.TIF') self.assertEquals(output, ["Image Processing Completed", 0]) @@ -148,8 +151,8 @@ def test_download_process_continuous_with_wrong_args(self, mock_downloader, mock args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.mock_path, '-p', '-u', '--region', 'whatever'] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2']) - mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False) + mock_downloader.assert_called_with(['LC80010092015051LGN00'], [4, 3, 2]) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False, False) self.assertEquals(output, ['Could not authenticate with AWS', 1]) @mock.patch('landsat.landsat.process_image') @@ -160,7 +163,8 @@ def test_process_correct(self, mock_process): args = ['process', 'path/to/folder/LC80010092015051LGN00'] output = landsat.main(self.parser.parse_args(args)) - mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, False, False) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, + False, False, False, False, False) self.assertEquals(output, ["The output is stored at image.TIF"]) @mock.patch('landsat.landsat.process_image') @@ -171,7 +175,18 @@ def test_process_correct_pansharpen(self, mock_process): args = ['process', '--pansharpen', 'path/to/folder/LC80010092015051LGN00'] output = landsat.main(self.parser.parse_args(args)) - mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, True, False) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, True, False, False, False) + self.assertEquals(output, ["The output is stored at image.TIF"]) + + @mock.patch('landsat.landsat.process_image') + def test_process_correct_ndvi(self, mock_process): + """Test process command with correct input and ndvi""" + mock_process.return_value = 'image.TIF' + + args = ['process', '--ndvi', 'path/to/folder/LC80010092015051LGN00'] + output = landsat.main(self.parser.parse_args(args)) + + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, False, True, False, False) self.assertEquals(output, ["The output is stored at image.TIF"]) def test_process_incorrect(self): diff --git a/landsat/tests/test_mixins.py b/tests/test_mixins.py similarity index 92% rename from landsat/tests/test_mixins.py rename to tests/test_mixins.py index eafee94..eaa5831 100644 --- a/landsat/tests/test_mixins.py +++ b/tests/test_mixins.py @@ -3,18 +3,12 @@ """Tests for mixins""" -import os import sys import unittest from cStringIO import StringIO from contextlib import contextmanager -try: - from landsat.mixins import VerbosityMixin - -except ImportError: - sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../landsat'))) - from landsat.mixins import VerbosityMixin +from landsat.mixins import VerbosityMixin # Capture function is taken from diff --git a/landsat/tests/test_search.py b/tests/test_search.py similarity index 94% rename from landsat/tests/test_search.py rename to tests/test_search.py index d2d5d6e..0779108 100644 --- a/landsat/tests/test_search.py +++ b/tests/test_search.py @@ -3,15 +3,9 @@ """Tests for search""" -import os -import sys import unittest -try: - from landsat.search import Search -except ImportError: - sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../landsat'))) - from landsat.search import Search +from landsat.search import Search class TestSearchHelper(unittest.TestCase): diff --git a/landsat/tests/test_uploader.py b/tests/test_uploader.py similarity index 76% rename from landsat/tests/test_uploader.py rename to tests/test_uploader.py index ac7e712..055c86d 100644 --- a/landsat/tests/test_uploader.py +++ b/tests/test_uploader.py @@ -12,20 +12,15 @@ import mock -try: - from landsat.uploader import Uploader, upload, upload_part, data_collector - import landsat.tests.mocks as mocks -except ImportError: - sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../landsat'))) - from landsat.uploader import Uploader - import landsat.tests.mocks as mocks +from landsat.uploader import Uploader, upload, upload_part, data_collector +from .mocks import S3Connection, state class TestUploader(unittest.TestCase): - @mock.patch('landsat.uploader.S3Connection', mocks.S3Connection) + @mock.patch('landsat.uploader.S3Connection', S3Connection) def test_upload_to_s3(self): - mocks.state['mock_boto_s3_multipart_upload_data'] = [] + state['mock_boto_s3_multipart_upload_data'] = [] base_dir = os.path.abspath(os.path.dirname(__file__)) landsat_image = os.path.join(base_dir, 'samples/mock_upload') f = open(landsat_image, 'rb').readlines() @@ -33,17 +28,17 @@ def test_upload_to_s3(self): u = Uploader('some_key', 'some_secret') u.run('some bucket', 'mock_upload', landsat_image) - self.assertEqual(mocks.state['mock_boto_s3_multipart_upload_data'], f) + self.assertEqual(state['mock_boto_s3_multipart_upload_data'], f) class upload_tests(unittest.TestCase): def test_should_be_able_to_upload_data(self): input = ['12', '345'] - mocks.state['mock_boto_s3_multipart_upload_data'] = [] - conn = mocks.S3Connection('some_key', 'some_secret', True) + state['mock_boto_s3_multipart_upload_data'] = [] + conn = S3Connection('some_key', 'some_secret', True) upload('test_bucket', 'some_key', 'some_secret', input, 'some_key', connection=conn) - self.assertEqual(mocks.state['mock_boto_s3_multipart_upload_data'], ['12', '345']) + self.assertEqual(state['mock_boto_s3_multipart_upload_data'], ['12', '345']) class upload_part_tests(unittest.TestCase): diff --git a/landsat/tests/test_utils.py b/tests/test_utils.py similarity index 89% rename from landsat/tests/test_utils.py rename to tests/test_utils.py index 7e2f4ac..b42c28b 100644 --- a/landsat/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,18 +3,13 @@ """Tests for utils""" -from os.path import join, abspath, dirname -import sys +from os.path import join import errno import shutil import unittest from tempfile import mkdtemp, mkstemp -try: - from landsat import utils -except ImportError: - sys.path.append(abspath(join(dirname(__file__), '../landsat'))) - import utils +from landsat import utils class TestUtils(unittest.TestCase): @@ -92,18 +87,24 @@ def test_reformat_date(self): def test_convert_to_integer_list(self): # correct input r = utils.convert_to_integer_list('1,2,3') - self.assertEqual(['1', '2', '3'], r) + self.assertEqual([1, 2, 3], r) # try other cobinations r = utils.convert_to_integer_list('1, 2, 3') - self.assertEqual(['1', '2', '3'], r) + self.assertEqual([1, 2, 3], r) r = utils.convert_to_integer_list('1s,2df,3d/') - self.assertEqual(['1', '2', '3'], r) + self.assertEqual([1, 2, 3], r) r = utils.convert_to_integer_list([1, 3, 4]) self.assertEqual([1, 3, 4], r) + r = utils.convert_to_integer_list('1,11,10') + self.assertEqual([1, 11, 10], r) + + r = utils.convert_to_integer_list('1,11,10,QA') + self.assertEqual([1, 11, 10, 'QA'], r) + if __name__ == '__main__': unittest.main()