Merge pull request #27 from USDA-ARS-NWRC/file_loading

HRRR module

CONTRIBUTING.rst

@@ -112,3 +112,15 @@ To run a subset of tests::
 
 
     $ python -m unittest tests.test_weather_forecast_retrieval
+
+
+Skip running tests with external HTTP requests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+To speed up development using local data, all tests that require an external
+HTTP request can be skipped via environment variable. To use this option, set
+
+::
+
+    WFR_SKIP_EXTERNAL_REQUEST_TEST=1
+
+as global variable in the environment that executes the tests.

requirements_dev.txt

@@ -4,4 +4,5 @@ coverage==5.0.3
 pylint==2.4.4
 flake8
 autopep8
-isort==4.3.21
+isort==4.3.21
+mock

scripts/convert_grib2nc

@@ -1,14 +1,12 @@
 #!/usr/bin/env python
 
 import argparse
-import logging
 import os
-from datetime import datetime, time
-from logging.handlers import TimedRotatingFileHandler
+from datetime import datetime
 
 import pandas as pd
 
-from weather_forecast_retrieval import hrrr, utils
+from weather_forecast_retrieval import utils
 from weather_forecast_retrieval.grib2nc import grib2nc
 
 
@@ -25,27 +23,16 @@ def run(config_file, start_date):
         ex_date = datetime.date(pd.to_datetime(start_date))
         date_folder = 'hrrr.{}'.format(ex_date.strftime('%Y%m%d'))
 
-        # setup logging
-        logfile = None
+        log_config = None
         if 'grib2nc_log' in config['logging']:
-            logfile = config['logging']['grib2nc_log']
-
-        fmt = '%(levelname)s:%(name)s:%(message)s'
-        log = logging.getLogger('grib2nc')
-        numeric_level = 1
-        if logfile is not None:
-            handler = TimedRotatingFileHandler(logfile,
-                                                when='D',
-                                                interval=1,
-                                                utc=True,
-                                                atTime=time(),
-                                                backupCount=30)
-            log.setLevel(numeric_level)
-            formatter = logging.Formatter(fmt)
-            handler.setFormatter(formatter)
-            log.addHandler(handler)
-        else:
-            logging.basicConfig(level=numeric_level)
+            log_config = {
+                'logging': {
+                    'log_file': config['logging']['grib2nc_log'],
+                    'log_level': 'ERROR',
+                }
+            }
+
+        log = utils.setup_local_logger('grib2nc', log_config)
 
         msg = "GRIB2NC Converter Utility"
         log.info(msg)
@@ -57,7 +44,7 @@ def run(config_file, start_date):
         if not os.path.isdir(grib2_path):
             log.warning('Grib2 path does not exist {}'.format(grib2_path))
             return True
-        
+
         # make and check the netcdf path
         nc_path = os.path.join(output_nc, date_folder)
         if not os.path.isdir(nc_path):
@@ -84,7 +71,7 @@ if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Convert folder of HRRR grib2 files to netcdf')
     parser.add_argument('config_file', metavar='config_file', type=str,
                         help='Path to config file')
-    
+
     parser.add_argument('-s', '--start', dest='start_date',
                         required=True, default=None,
                         help='Date to convert')

scripts/run_hrrr_retrieval

@@ -3,7 +3,7 @@
 import argparse
 import os
 
-from weather_forecast_retrieval import hrrr
+from weather_forecast_retrieval.data.hrrr import HttpRetrieval
 
 
 def run():
@@ -15,9 +15,7 @@ def run():
     args = parser.parse_args()
 
     if os.path.isfile(args.config_file):
-        #hrrr.HRRR(args.config_file).retrieve_grib_filter()
-        # hrrr.HRRR(args.config_file).retrieve_ftp()
-        hrrr.HRRR(args.config_file).retrieve_http_by_date()
+        HttpRetrieval(args.config_file).fetch_by_date()
 
     else:
         raise IOError('File does not exist.')

scripts/run_hrrr_retrieval_dates

@@ -3,15 +3,17 @@
 import argparse
 import os
 
-from weather_forecast_retrieval import hrrr
+from weather_forecast_retrieval.data.hrrr import HttpRetrieval
 
 
 def run():
+    parser = argparse.ArgumentParser(
+        description='Run the data retrieval for HRRR model'
+    )
 
-    parser = argparse.ArgumentParser(description='Run the data retrieval for HRRR model')
     parser.add_argument('config_file', metavar='config_file', type=str,
                         help='Path to config file')
-    
+
     parser.add_argument('-s', '--start', dest='start_date',
                         required=False, default=None,
                         help='Start date to look for modified files')
@@ -23,8 +25,9 @@ def run():
     args = parser.parse_args()
 
     if os.path.isfile(args.config_file):
-        hrrr.HRRR(args.config_file).retrieve_http_by_date(args.start_date, args.end_date)
-
+        HttpRetrieval(args.config_file).fetch_by_date(
+            args.start_date, args.end_date
+        )
     else:
         raise IOError('File does not exist.')
 

tests/hrrr_dates_test.ini → tests/RME/hrrr_dates_test.ini

@@ -3,8 +3,8 @@
 # Output
 ################################################################################
 [output]
-output_dir: tests/RME/output
-output_nc: tests/RME/output
+output_dir: ./output
+output_nc: ./output
 start_date: '7/10/2019 09:00'
 end_date: '7/10/2019 10:00'
 num_requests: 5

tests/RME_test_case.py

@@ -0,0 +1,39 @@
+import os
+import shutil
+import unittest
+from pathlib import Path
+
+import pandas as pd
+
+
+class RMETestCase(unittest.TestCase):
+    """
+    Base class for all unit tests using the RME data
+    """
+    test_dir = Path(__file__).parent
+    basin_dir = test_dir.joinpath('RME')
+    gold_dir = basin_dir.joinpath('gold', 'hrrr')
+    hrrr_dir = basin_dir.joinpath('gridded/hrrr_test')
+    output_path = basin_dir.joinpath('output')
+
+    START_DATE = pd.to_datetime('2018-07-22 01:00')
+    END_DATE = pd.to_datetime('2018-07-22 06:00')
+    UTM_ZONE_NUMBER = 11
+    BBOX = [-116.85837324, 42.96134124, -116.64913327, 43.16852535]
+
+    def setUp(self):
+        """
+        Create test directory structure
+        Change directory to RME test
+        """
+        self.output_path.mkdir(exist_ok=True)
+        os.chdir(self.basin_dir.as_posix())
+
+    def tearDown(self):
+        """
+        Cleanup the downloaded files
+        Cleanup grib2 index files
+        """
+        shutil.rmtree(self.output_path)
+        for index_file in self.hrrr_dir.rglob('**/*.grib2.*.idx'):
+            index_file.unlink()

tests/data/hrrr/__init__.py

@@ -0,0 +1,6 @@
+"""
+The High Resolution Rapid Refresh (HRRR) model is a NOAA
+real-time 3 km resolution forecast. The model output is updated
+hourly and assimilates 3km radar. More information can be found
+at https://rapidrefresh.noaa.gov/hrrr/.
+"""

tests/data/hrrr/test_base_file.py

@@ -0,0 +1,31 @@
+import unittest
+
+from weather_forecast_retrieval.data.hrrr.base_file import BaseFile
+
+
+class TestBaseFile(unittest.TestCase):
+    LOGGER_NAME = 'BaseFileTest'
+
+    @classmethod
+    def setUpClass(cls):
+        cls.subject = BaseFile(cls.LOGGER_NAME)
+
+    def test_bbox_property(self):
+        self.assertIsNone(self.subject.bbox)
+
+        self.subject.bbox = []
+
+        self.assertEqual([], self.subject.bbox)
+
+    def test_has_log_property(self):
+        self.assertEqual(self.LOGGER_NAME, self.subject.log.name)
+
+    def test_variable_map_property(self):
+        self.assertEqual(
+            BaseFile.VAR_MAP,
+            self.subject.variable_map
+        )
+
+    def test_load_method(self):
+        with self.assertRaises(NotImplementedError):
+            self.subject.load(None)

tests/data/hrrr/test_config_file.py

@@ -0,0 +1,67 @@
+import logging
+import unittest
+
+import mock
+import pandas as pd
+
+from weather_forecast_retrieval.data.hrrr.config_file import ConfigFile
+
+
+class TestConfigFile(unittest.TestCase):
+    LOGGER_NAME = 'ConfigFileTest'
+    CONFIG = {
+        'output': {
+            'output_dir': 'output_location',
+            'start_date': '2020-12-31 00:00',
+            'end_date': '2020-12-31 23:00',
+        }
+    }
+
+    @classmethod
+    def setUpClass(cls):
+        with mock.patch(
+            'weather_forecast_retrieval.utils.read_config',
+            return_value=cls.CONFIG
+        ):
+            cls.subject = ConfigFile(cls.LOGGER_NAME, cls.CONFIG)
+
+    def test_output_dir(self):
+        self.assertEqual(
+            self.CONFIG['output']['output_dir'],
+            self.subject.output_dir
+        )
+
+    def test_start_date(self):
+        self.assertEqual(
+            pd.to_datetime(self.CONFIG['output']['start_date']),
+            self.subject.start_date,
+        )
+
+    def test_end_date(self):
+        self.assertEqual(
+            pd.to_datetime(self.CONFIG['output']['end_date']),
+            self.subject.end_date
+        )
+
+    def test_no_config(self):
+        subject = ConfigFile(self.LOGGER_NAME)
+        self.assertIsNone(subject.config)
+
+    def test_default_properties(self):
+        subject = ConfigFile(self.LOGGER_NAME)
+        self.assertIsNone(subject.start_date)
+        self.assertIsNone(subject.end_date)
+        self.assertIsNone(subject.output_dir)
+
+    def test_logger_name(self):
+        self.assertEqual(self.LOGGER_NAME, self.subject.log.name)
+
+    def test_external_logger(self):
+        external_logger = logging.Logger('External')
+        subject = ConfigFile(
+            self.LOGGER_NAME, external_logger=external_logger
+        )
+        self.assertEqual(external_logger.name, subject.log.name)
+
+    def test_log_property(self):
+        self.assertIsInstance(self.subject.log, logging.Logger)

tests/data/hrrr/test_file_handler.py

@@ -0,0 +1,55 @@
+import unittest
+
+import pandas as pd
+
+from weather_forecast_retrieval.data.hrrr import FileHandler
+
+
+class TestHRRRFileHandler(unittest.TestCase):
+    def test_file_date(self):
+        file_time = pd.to_datetime('2018-02-08 05:00')
+
+        forecast_hour = 1
+        day, file_hour = FileHandler.file_date(
+            file_time, forecast_hour
+        )
+        self.assertEqual('2018-02-08', str(day))
+        self.assertEqual(4, file_hour)
+
+        forecast_hour = 3
+        day, file_hour = FileHandler.file_date(
+            file_time, forecast_hour
+        )
+        self.assertEqual('2018-02-08', str(day))
+        self.assertEqual(2, file_hour)
+
+        forecast_hour = 8
+        day, file_hour = FileHandler.file_date(
+            file_time, forecast_hour
+        )
+        self.assertEqual('2018-02-07', str(day))
+        self.assertEqual(21, file_hour)
+
+    def test_file_name(self):
+        self.assertEqual(
+            'hrrr.t04z.wrfsfcf01.grib2',
+            FileHandler.file_name(4, 1)
+        )
+        self.assertEqual(
+            'hrrr.t04z.wrfsfcf01.nc',
+            FileHandler.file_name(4, 1, 'netcdf')
+        )
+
+    def test_folder_name(self):
+        self.assertEqual(
+            'hrrr.20180208',
+            FileHandler.folder_name(pd.to_datetime('2018-02-08'))
+        )
+
+    def test_folder_and_file(self):
+        folder_name, file_name = FileHandler.folder_and_file(
+            pd.to_datetime('2018-02-08 05:00'), 1
+        )
+
+        self.assertEqual('hrrr.20180208', folder_name)
+        self.assertEqual('hrrr.t04z.wrfsfcf01.grib2', file_name)