Merge branch 'develop'

.travis.yml

@@ -1,27 +1,46 @@
 language: python
 python:
   - 2.7
+sudo: false
 
-# Setup anaconda
+# Setup miniconda
 before_install:
-  - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
-  - chmod +x miniconda.sh
-  - ./miniconda.sh -b
-  - export PATH=/home/travis/miniconda/bin:$PATH
+  - if [[ ! -e $HOME/miniconda/bin ]]; then
+        wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh;
+        chmod +x miniconda.sh;
+        ./miniconda.sh -b -f -p $HOME/miniconda;
+    fi
+  - export OLD_PATH=$PATH
+  - export PATH=$HOME/miniconda/bin:$PATH
 
 install:
   - conda update -q --yes conda
-  - conda install -q --yes numpy scipy matplotlib nose
-  - conda install -q --yes -c https://conda.binstar.org/osgeo gdal
+  - conda install -q --yes numpy scipy matplotlib nose pillow basemap netcdf4 proj.4
+  - conda install -q --yes -c https://conda.anaconda.org/nersc nansat-gdal
   - export GDAL_DATA=/home/vagrant/miniconda/share/gdal/
   - export GEOS_DIR=/home/vagrant/miniconda/
-  - pip install -q basemap --allow-external basemap --allow-unverified basemap
   - python -c 'import gdal; print gdal.__file__'
   - python -c 'from mpl_toolkits.basemap import Basemap'
   - pip install coveralls
-#- python setup.py install
+  - pip install cfunits
+  - python setup.py install
 
 script: #"cd ..; nosetests --with-coverage --cover-package=nansat nansat.tests"
-  coverage run --source=nansat setup.py test
+  - coverage run --source=nansat setup.py test
+
 after_success:
-  coveralls
+  - coveralls
+  # Resetting path so uploading of cache with curl succeeds.
+  - export PATH=$OLD_PATH
+
+after_failure:
+  # Resetting path so uploading of cache with curl succeeds.
+  - export PATH=$OLD_PATH
+
+# before_cache:
+#   - rm -f $HOME/.cache/pip/log/debug.log
+
+cache:
+  directories:
+#     - $HOME/.cache/pip
+    - $HOME/miniconda

README.md

@@ -1,11 +1,9 @@
-[![Build Status](https://travis-ci.org/nansencenter/nansat.svg?branch=master)](https://travis-ci.org/nansencenter/nansat)
-[![Coverage Status](https://coveralls.io/repos/nansencenter/nansat/badge.svg?branch=master)](https://coveralls.io/r/nansencenter/nansat?branch=master)
+[![Build Status](https://travis-ci.org/nansencenter/nansat.svg?branch=develop)](https://travis-ci.org/nansencenter/nansat)
+[![Coverage Status](https://coveralls.io/repos/nansencenter/nansat/badge.svg?branch=develop)](https://coveralls.io/r/nansencenter/nansat)
 
 ![NANSAT](http://nansencenter.github.io/nansat/images/nansat_logo.png)
-nansat
-======
 
-Nansat is a scientist friendly Python toolbox for processing 2D satellite earth observation data.
+**Nansat** is a scientist friendly Python toolbox for processing 2D satellite earth observation data.
 
 The main **goal** of Nansat is to facilitate:
 
@@ -17,21 +15,64 @@ We appreciate acknowledments of Nansat. Please add "The image analysis was perfo
 the open-source NanSat (https://github.com/nansencenter/nansat) python package" (or equivalent)
 if you use Nansat in scientific publications.
 
-Download
-=========
-https://github.com/nansencenter/nansat/wiki/Download
+## Easy to install
+The easiest way to install Nansat on a Linux machine is to use [anaconda](http://docs.continuum.io/anaconda/index)
+```
+# download lates version of miniconda
+wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
 
+# make it executable
+chmod +x miniconda.sh
 
-Install Nansat
-==============
-https://github.com/nansencenter/nansat/wiki/Install-Nansat
+# install miniconda virtual environment
+./miniconda.sh -b -f -p $HOME/miniconda
 
+# activate the environment 
+export PATH=$HOME/miniconda/bin/:$PATH
 
-Install required libraries
-==========================
-See https://github.com/nansencenter/nansat/wiki/Required-libs
+# install some requirements from common repositories
+conda install -q --yes numpy scipy matplotlib nose pillow basemap netcdf4 proj.4
 
+#install some requirements from NERSC repository
+conda install -q --yes -c https://conda.anaconda.org/nersc nansat-gdal
 
-Tutorial
-========
-https://github.com/nansencenter/nansat/wiki/Tutorial
+# install some requiremets from pypi
+pip install cfunits
+
+# configure environment
+export GDAL_DATA=$HOME/miniconda/share/gdal/
+export GEOS_DIR=$HOME/miniconda/
+
+# install nersc-metadata which is not in pip yet
+pip install https://github.com/nansencenter/nersc-metadata/archive/master.tar.gz
+
+# finally install Nansat
+pip install https://github.com/nansencenter/nersc-metadata/archive/develop.tar.gz
+
+# run tests
+nosetests nansat
+```
+Fore more information see [Install-Nansat](https://github.com/nansencenter/nansat/wiki/Install-Nansat) section or
+use pre-configure virtual machines as explained on [Nansat-lectures](https://github.com/nansencenter/nansat-lectures)
+
+## Easy to use
+```Python
+# download a test file
+!wget https://github.com/nansencenter/nansat/raw/develop/nansat/tests/data/stere.tif
+
+# import main file opener
+from nansat import Nansat
+
+# open a test file
+n = Nansat('stere.tif')
+
+# see file content
+print n
+
+# view file footpring
+n.write_map('stere.footpring.png')
+
+# create RGB with auto-stretched histogram
+n.write_figure('stere_rgb.png', [1,2,3], clim='hist')
+```
+Fore more information see [Tutorial](https://github.com/nansencenter/nansat/wiki/Tutorial) or notebooks for [Nansat lectures](https://github.com/nansencenter/nansat-lectures/tree/master/notebooks)

end2endtests/mapper_test_archive.py

@@ -0,0 +1,62 @@
+#-------------------------------------------------------------------------------
+# Name:         test_nansat_archive.py
+# Purpose:      To test nansat
+#
+# Author:       Anton Korosov, Morten Wergeland Hansen, Asuka Yamakawa
+# Modified: Morten Wergeland Hansen
+#
+# Created:  18.06.2014
+# Last modified:03.06.2015 13:36
+# Copyright:    (c) NERSC
+# Licence:      This file is part of NANSAT. You can redistribute it or modify
+#               under the terms of GNU General Public License, v.3
+#               http://www.gnu.org/licenses/gpl-3.0.html
+#-------------------------------------------------------------------------------
+import os
+import warnings
+import time
+import glob
+
+
+class DataForTestingMappers(object):
+    def __init__(self):
+        ''' Find test files and corresponsing mapper names '''
+        existingTestFiles = self.find_existing_files()
+        filesAndMappers = self.identify_mappers(existingTestFiles)
+
+        self.mapperData = filesAndMappers
+
+    def find_existing_files(self):
+        ''' Find all files for testsing inside MAPPER_TEST_DATA_DIR'''
+        testFiles = []
+
+        testDataEnv = os.getenv('MAPPER_TEST_DATA_DIR')
+        if testDataEnv is not None:
+            testDataDirs = testDataEnv.split(':')
+            for testDataDir in testDataDirs:
+                if os.path.isdir(testDataDir):
+                    testFiles += glob.glob(os.path.join(testDataDir, '*', '*'))
+
+        testFiles = [testFile for testFile in testFiles if self.readable(testFile)]
+
+        return testFiles
+
+    def identify_mappers(self, testFiles):
+        ''' From the sub-directory name get the name of the mapper '''
+
+        mapperNames = [os.path.split(os.path.split(testFile)[0])[1] for testFile in testFiles]
+        return zip(testFiles, mapperNames)
+
+
+    def readable(self, testFile):
+        ''' Test if file is readable at OS level '''
+        if not os.path.exists(testFile):
+            return False
+        if not os.access(testFile, os.R_OK):
+            return False
+        if os.stat(testFile).st_size == 0:
+            return False
+        if os.path.isdir(testFile):
+            return False
+
+        return True

end2endtests/test_mappers.py

@@ -0,0 +1,139 @@
+#-------------------------------------------------------------------------------
+# Name:         test_nansat.py
+# Purpose:      Test the nansat module
+#
+# Author:       Morten Wergeland Hansen, Asuka Yamakawa, Anton Korosov
+# Modified: Morten Wergeland Hansen
+#
+# Created:      18.06.2014
+# Last modified:02.07.2015 16:05
+# Copyright:    (c) NERSC
+# Licence:      This file is part of NANSAT. You can redistribute it or modify
+#               under the terms of GNU General Public License, v.3
+#               http://www.gnu.org/licenses/gpl-3.0.html
+#-------------------------------------------------------------------------------
+import unittest, warnings
+import os, sys, glob, datetime
+import json
+
+from types import ModuleType, FloatType
+import numpy as np
+
+from nerscmetadata import gcmd_keywords
+
+from nansat import Nansat, Domain
+from nansat.nansat import _import_mappers
+from mapper_test_archive import DataForTestingMappers
+
+nansatMappers = _import_mappers()
+
+class TestDataForTestingMappers(unittest.TestCase):
+    def test_create_test_data(self):
+        ''' should create TestData instance '''
+        t = DataForTestingMappers()
+        self.assertTrue(hasattr(t, 'mapperData'))
+
+# https://nose.readthedocs.org/en/latest/writing_tests.html#test-generators
+# The x-flag results in the test stopping at first failure or error - use it
+# for easier debugging:
+# nosetests -v -x end2endtests.test_mappers:TestAllMappers.test_mappers_basic
+class TestAllMappers(object):
+
+    @classmethod
+    def setup_class(cls):
+        ''' Download testing data '''
+        cls.testData = DataForTestingMappers()
+
+    def test_mappers_basic(self):
+        ''' Run similar basic tests for all mappers '''
+        for fileName, mapperName in self.testData.mapperData:
+            sys.stderr.write('\nMapper '+mapperName+' -> '+fileName+'\n')
+            # Test call to Nansat, mapper not specified
+            yield self.open_with_nansat, fileName
+            # Test call to Nansat, mapper specified
+            yield self.open_with_nansat, fileName, mapperName
+
+    def test_mappers_start_time(self):
+        ''' Run similar NansenCloud reated tests for all mappers '''
+        for fileName, mapperName in self.testData.mapperData:
+            sys.stderr.write('\nMapper '+mapperName+' -> '+fileName+'\n')
+            n = Nansat(fileName, mapperName=mapperName)
+            # Test nansat.start_time() and nansat.end_time()
+            yield self.has_start_time, n
+
+    def test_mappers_advanced(self):
+        ''' Run similar NansenCloud reated tests for all mappers '''
+        for fileName, mapperName in self.testData.mapperData:
+            sys.stderr.write('\nMapper '+mapperName+' -> '+fileName+'\n')
+            n = Nansat(fileName, mapperName=mapperName)
+            yield self.is_correct_mapper, n, mapperName
+            yield self.has_start_time, n
+            yield self.has_end_time, n
+            yield self.has_correct_platform, n
+            yield self.has_correct_instrument, n
+
+            # Test that SAR objects have sigma0 intensity bands in addition
+            # to complex bands
+            if n.has_band(
+                'surface_backwards_scattering_coefficient_of_radar_wave'
+                    ):
+                yield self.exist_intensity_band, n
+
+    def has_start_time(self, n):
+        ''' Has start time '''
+        assert type(n.time_coverage_start)==datetime.datetime
+
+    def has_end_time(self, n):
+        assert type(n.time_coverage_end)==datetime.datetime
+
+    def has_correct_platform(self, n):
+        meta1 = json.loads(n.get_metadata('platform'))
+        meta1ShortName = meta1['Short_Name']
+        meta2 = gcmd_keywords.get_platform(meta1ShortName)
+
+        assert type(meta1) == dict
+        assert meta1 == meta2
+
+    def has_correct_instrument(self, n):
+        meta1 = json.loads(n.get_metadata('instrument'))
+        meta1ShortName = meta1['Short_Name']
+        meta2 = gcmd_keywords.get_instrument(meta1ShortName)
+
+        assert type(meta1) == dict
+        assert meta1 == meta2
+
+    def is_correct_mapper(self, n, mapper):
+        assert n.mapper==mapper
+
+    def open_with_nansat(self, file, mapper=None, kwargs=None):
+        ''' Perform call to Nansat and check that it returns a Nansat object '''
+        if kwargs is None:
+            kwargs = {}
+
+        if mapper:
+            n = Nansat(file, mapperName=mapper, **kwargs)
+        else:
+            n = Nansat(file, **kwargs)
+        assert type(n) == Nansat
+
+    def exist_intensity_band(self, n):
+        ''' test if intensity bands exist for complex data '''
+        allBandNames = []
+        complexBandNames = []
+        for iBand in range(n.vrt.dataset.RasterCount):
+            iBandName = n.get_metadata(bandID=iBand + 1)['name']
+            allBandNames.append(iBandName)
+            if '_complex' in iBandName:
+                complexBandNames.append(iBandName)
+
+        for iComplexName in complexBandNames:
+            assert iComplexName.replace('_complex', '') in allBandNames
+
+if __name__=='__main__':
+    #for mapper in nansatMappers:
+    #    test_name = 'test_%s'%mapper
+    unittest.main()
+
+
+
+