Merge pull request #40 from Open-ET/tcorr-index-nine

Set tcorr index to 9 if coarse count is 0

openet/ssebop/__init__.py

@@ -2,6 +2,6 @@
 from .collection import Collection
 from . import interpolate
 
-__version__ = "0.1.6"
+__version__ = "0.1.7"
 
 MODEL_NAME = 'SSEBOP'

openet/ssebop/image.py

@@ -1632,9 +1632,13 @@ def tcorr_gridded(self):
         quality_score_img = ee.Image([total_score_img, hotscore, coldscore, cold_rn05_score])\
             .reduce(ee.Reducer.sum())
 
+        # Set tcorr index to 9 if coarse count is 0
+        # This if should be fast but the calculation below works also
+        tcorr_index = ee.Algorithms.If(tcorr_count_cold.gt(0), 0, 9)
+
         return ee.Image([tcorr, quality_score_img]).rename(['tcorr', 'quality'])\
             .set(self._properties)\
-            .set({'tcorr_index': 0,
+            .set({'tcorr_index': tcorr_index,
                   'tcorr_coarse_count_cold': tcorr_count_cold})
 
     @lazy_property
@@ -1781,7 +1785,12 @@ def tcorr_gridded_cold(self):
             .reproject(crs=self.crs, crsTransform=coarse_transform)\
             .updateMask(1)
 
+        # Set tcorr index to 9 if coarse count is 0
+        # This "if" should be fast but the calculation approach works also
+        tcorr_index = ee.Algorithms.If(tcorr_count.gt(0), 1, 9)
+        # tcorr_index = tcorr_count.multiply(-1).max(-1).add(1).multiply(8).add(1)
+
         return ee.Image([tcorr, total_score_img]).rename(['tcorr', 'quality']) \
             .set(self._properties) \
-            .set({'tcorr_index': 1,
+            .set({'tcorr_index': tcorr_index,
                   'tcorr_coarse_count': tcorr_count})

openet/ssebop/tests/test_c_image.py

@@ -1065,6 +1065,11 @@ def test_Image_tcorr_gridded_method(tcorr_source, tmax_source, image_id,
          'LANDSAT/LC08/C02/T1_L2/LC08_044033_20170716',
          [600000, 4270000, 625000, 4285000], [612500, 4277500],
          [0.9914034763259638, 1]],
+        # TODO: Add a test where coarse count is 0
+        ['GRIDDED_COLD', 'projects/usgs-ssebop/tmax/daymet_v4_mean_1981_2010_elr',
+         'LANDSAT/LC08/C02/T1_L2/LC08_044033_20200910',
+         [600000, 4270000, 625000, 4285000], [612500, 4277500],
+         [None, 9]],
     ]
 )
 def test_Image_tcorr_gridded_cold_method(tcorr_source, tmax_source, image_id,
@@ -1080,7 +1085,10 @@ def test_Image_tcorr_gridded_cold_method(tcorr_source, tmax_source, image_id,
         tcorr_resample='nearest').tcorr_gridded_cold
     tcorr = utils.point_image_value(tcorr_img, point_xy)
     index = utils.getinfo(tcorr_img.get('tcorr_index'))
-    assert abs(tcorr['tcorr'] - expected[0]) <= tol
+    if expected[0] is None:
+        assert tcorr['tcorr'] is None
+    else:
+        assert abs(tcorr['tcorr'] - expected[0]) <= tol
     assert index == expected[1]
 
 

tcorr_gridded/ini/tcorr_daymet_v4_monthly.ini

@@ -36,8 +36,8 @@ export_coll = projects/earthengine-legacy/assets/projects/usgs-ssebop/tcorr_grid
 # mgrs_tiles = 10S, 10T, 11S
 # utm_zones = 10, 11, 12, 13, 14, 15
 
-# mgrs_ftr_coll = projects/earthengine-legacy/assets/projects/openet/mgrs/conus_gridmet/zones
-mgrs_ftr_coll = projects/earthengine-legacy/assets/projects/openet/mgrs/conus_gridmet/tiles
+mgrs_ftr_coll = projects/earthengine-legacy/assets/projects/openet/mgrs/conus_gridmet/zones
+# mgrs_ftr_coll = projects/earthengine-legacy/assets/projects/openet/mgrs/conus_gridmet/tiles
 
 
 [SSEBOP]

tcorr_gridded/tcorr_export_monthly.py

@@ -547,15 +547,23 @@ def mgrs_export_tiles(study_area_coll_id, mgrs_coll_id,
     logging.debug('Building study area collection')
     logging.debug(f'  {study_area_coll_id}')
     study_area_coll = ee.FeatureCollection(study_area_coll_id)
-    if (study_area_property == 'STUSPS' and
+    if (study_area_coll_id in ['TIGER/2018/States'] and
             'CONUS' in [x.upper() for x in study_area_features]):
-        # Exclude AK, HI, AS, GU, PR, MP, VI, (but keep DC)
+        study_area_coll_id = 'projects/openet/featureCollections/boundaries/tiger_2018_conus'
+        study_area_coll = ee.FeatureCollection(study_area_coll_id)
+        study_area_property == 'STUSPS'
+        study_area_features = ['CONUS']
+    elif (study_area_property == 'STUSPS' and
+            'CONUS' in [x.upper() for x in study_area_features]):
+        # Excluding AK, HI, AS, GU, PR, MP, VI
+        # Dropping some other states also since they aren't needed in order
+        #   to intersect all the MGRS tiles in the CONUS (CT, DE, DC, RI, VT)
         study_area_features = [
-            'AL', 'AR', 'AZ', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA',
+            'AL', 'AR', 'AZ', 'CA', 'CO', 'FL', 'GA',
             'IA', 'ID', 'IL', 'IN', 'KS', 'KY', 'LA', 'MA', 'MD', 'ME',
             'MI', 'MN', 'MO', 'MS', 'MT', 'NC', 'ND', 'NE', 'NH', 'NJ',
-            'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD',
-            'TN', 'TX', 'UT', 'VA', 'VT', 'WA', 'WI', 'WV', 'WY']
+            'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'SC', 'SD',
+            'TN', 'TX', 'UT', 'VA', 'WA', 'WI', 'WV', 'WY']
     # elif (study_area_property == 'STUSPS' and
     #         'WESTERN11' in [x.upper() for x in study_area_features]):
     #     study_area_features = [

tcorr_gridded/tcorr_export_scene_by_date.py

@@ -16,7 +16,7 @@
 import openet.core.utils as utils
 
 TOOL_NAME = 'tcorr_export_scene_by_date'
-TOOL_VERSION = '0.1.6'
+TOOL_VERSION = '0.1.7'
 
 # TODO: This could be a property or method of SSEBop or the Image class
 TCORR_INDICES = {
@@ -567,14 +567,6 @@ def version_number(version_str):
                 tcorr_img = t_obj.tcorr_gridded_cold
             # tcorr_img = t_obj.tcorr
 
-            # Properties that the climo tcorr image might change need to be set
-            #   before the climo is used
-            # It would probably make more sense to move all of the property
-            #   setting to right here instead of down below
-            tcorr_img = tcorr_img.set({
-                'tcorr_index': TCORR_INDICES[tcorr_source.upper()],
-            })
-
             # Replace masked tcorr images with climos
             # Note, If the month climo doesn't exist this will keep the
             #   existing masked Tcorr image (we may want to change that)
@@ -591,7 +583,7 @@ def version_number(version_str):
                     .addBands([tcorr_month_coll.first().multiply(0).rename(['quality'])])\
                     .set({'tcorr_coarse_count': None})
                 tcorr_img = ee.Algorithms.If(
-                    ee.Number(tcorr_img.get('tcorr_coarse_count')).eq(0)
+                    ee.Number(tcorr_img.get('tcorr_index')).eq(9)
                         .And(tcorr_month_coll.size().gt(0)),
                     tcorr_month_img,
                     tcorr_img)
@@ -635,7 +627,7 @@ def version_number(version_str):
                     'wrs2_tile': wrs2_tile,
                     'year': int(export_dt.year),
                 })
-            # pprint.pprint(output_img.getInfo()['properties'])
+            # pprint.pprint(tcorr_img.getInfo()['properties'])
             # input('ENTER')
 
             logging.debug('  Building export task')
@@ -776,15 +768,23 @@ def mgrs_export_tiles(study_area_coll_id, mgrs_coll_id,
     logging.debug('Building study area collection')
     logging.debug(f'  {study_area_coll_id}')
     study_area_coll = ee.FeatureCollection(study_area_coll_id)
-    if (study_area_property == 'STUSPS' and
+    if (study_area_coll_id in ['TIGER/2018/States'] and
+            'CONUS' in [x.upper() for x in study_area_features]):
+        study_area_coll_id = 'projects/openet/featureCollections/boundaries/tiger_2018_conus'
+        study_area_coll = ee.FeatureCollection(study_area_coll_id)
+        study_area_property == 'STUSPS'
+        study_area_features = ['CONUS']
+    elif (study_area_property == 'STUSPS' and
             'CONUS' in [x.upper() for x in study_area_features]):
-        # Exclude AK, HI, AS, GU, PR, MP, VI, (but keep DC)
+        # Excluding AK, HI, AS, GU, PR, MP, VI
+        # Dropping some other states also since they aren't needed in order
+        #   to intersect all the MGRS tiles in the CONUS (CT, DE, DC, RI, VT)
         study_area_features = [
-            'AL', 'AR', 'AZ', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA',
+            'AL', 'AR', 'AZ', 'CA', 'CO', 'FL', 'GA',
             'IA', 'ID', 'IL', 'IN', 'KS', 'KY', 'LA', 'MA', 'MD', 'ME',
             'MI', 'MN', 'MO', 'MS', 'MT', 'NC', 'ND', 'NE', 'NH', 'NJ',
-            'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD',
-            'TN', 'TX', 'UT', 'VA', 'VT', 'WA', 'WI', 'WV', 'WY']
+            'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'SC', 'SD',
+            'TN', 'TX', 'UT', 'VA', 'WA', 'WI', 'WV', 'WY']
     # elif (study_area_property == 'STUSPS' and
     #         'WESTERN11' in [x.upper() for x in study_area_features]):
     #     study_area_features = [

tcorr_gridded/tcorr_export_scene_by_wrs2.py

@@ -16,7 +16,7 @@
 import openet.core.utils as utils
 
 TOOL_NAME = 'tcorr_export_scene_by_wrs2'
-TOOL_VERSION = '0.1.6'
+TOOL_VERSION = '0.1.7'
 
 # TODO: This could be a property or method of SSEBop or the Image class
 TCORR_INDICES = {
@@ -571,14 +571,6 @@ def version_number(version_str):
                 tcorr_img = t_obj.tcorr_gridded_cold
             # tcorr_img = t_obj.tcorr
 
-            # Properties that the climo tcorr image might change need to be set
-            #   before the climo is used
-            # It would probably make more sense to move all of the property
-            #   setting to right here instead of down below
-            tcorr_img = tcorr_img.set({
-                'tcorr_index': TCORR_INDICES[tcorr_source.upper()],
-            })
-
             # Replace masked tcorr images with climos
             # Note, If the month climo doesn't exist this will keep the
             #   existing masked Tcorr image (we may want to change that)
@@ -595,7 +587,7 @@ def version_number(version_str):
                     .addBands([tcorr_month_coll.first().multiply(0).rename(['quality'])])\
                     .set({'tcorr_coarse_count': None})
                 tcorr_img = ee.Algorithms.If(
-                    ee.Number(tcorr_img.get('tcorr_coarse_count')).eq(0)
+                    ee.Number(tcorr_img.get('tcorr_index')).eq(9)
                         .And(tcorr_month_coll.size().gt(0)),
                     tcorr_month_img,
                     tcorr_img)
@@ -640,7 +632,7 @@ def version_number(version_str):
                     'wrs2_tile': wrs2_tile_fmt.format(wrs2_path, wrs2_row),
                     'year': int(export_dt.year),
                 })
-            # pprint.pprint(output_img.getInfo()['properties'])
+            # pprint.pprint(tcorr_img.getInfo()['properties'])
             # input('ENTER')
 
             logging.debug('  Building export task')
@@ -782,15 +774,23 @@ def mgrs_export_tiles(study_area_coll_id, mgrs_coll_id,
     logging.debug('Building study area collection')
     logging.debug(f'  {study_area_coll_id}')
     study_area_coll = ee.FeatureCollection(study_area_coll_id)
-    if (study_area_property == 'STUSPS' and
+    if (study_area_coll_id in ['TIGER/2018/States'] and
+            'CONUS' in [x.upper() for x in study_area_features]):
+        study_area_coll_id = 'projects/openet/featureCollections/boundaries/tiger_2018_conus'
+        study_area_coll = ee.FeatureCollection(study_area_coll_id)
+        study_area_property == 'STUSPS'
+        study_area_features = ['CONUS']
+    elif (study_area_property == 'STUSPS' and
             'CONUS' in [x.upper() for x in study_area_features]):
-        # Exclude AK, HI, AS, GU, PR, MP, VI, (but keep DC)
+        # Excluding AK, HI, AS, GU, PR, MP, VI
+        # Dropping some other states also since they aren't needed in order
+        #   to intersect all the MGRS tiles in the CONUS (CT, DE, DC, RI, VT)
         study_area_features = [
-            'AL', 'AR', 'AZ', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA',
+            'AL', 'AR', 'AZ', 'CA', 'CO', 'FL', 'GA',
             'IA', 'ID', 'IL', 'IN', 'KS', 'KY', 'LA', 'MA', 'MD', 'ME',
             'MI', 'MN', 'MO', 'MS', 'MT', 'NC', 'ND', 'NE', 'NH', 'NJ',
-            'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD',
-            'TN', 'TX', 'UT', 'VA', 'VT', 'WA', 'WI', 'WV', 'WY']
+            'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'SC', 'SD',
+            'TN', 'TX', 'UT', 'VA', 'WA', 'WI', 'WV', 'WY']
     # elif (study_area_property == 'STUSPS' and
     #         'WESTERN11' in [x.upper() for x in study_area_features]):
     #     study_area_features = [
@@ -804,7 +804,7 @@ def mgrs_export_tiles(study_area_coll_id, mgrs_coll_id,
         study_area_coll = study_area_coll.filter(
             ee.Filter.inList(study_area_property, study_area_features))
 
-    logging.info('Building MGRS tile list')
+    logging.debug('Building MGRS tile list')
     tiles_coll = ee.FeatureCollection(mgrs_coll_id) \
         .filterBounds(study_area_coll.geometry())
 

tcorr_gridded/tcorr_gridded_assets/main.py

@@ -16,7 +16,7 @@
 logging.getLogger('googleapiclient').setLevel(logging.ERROR)
 
 TOOL_NAME = 'tcorr_gridded_cloud_function'
-TOOL_VERSION = '0.1.0'
+TOOL_VERSION = '0.1.7'
 
 # TODO: Move all of these to config.py?
 FUNCTION_URL = 'https://us-central1-ssebop.cloudfunctions.net'
@@ -228,14 +228,6 @@ def tcorr_gridded_asset_ingest(image_id, overwrite_flag=True,
     tcorr_img = t_obj.tcorr_gridded_cold
     # tcorr_img = t_obj.tcorr
 
-    # Properties that the climo tcorr image might change need to be set
-    #   before the climo is used
-    # It would probably make more sense to move all of the property
-    #   setting to right here instead of down below
-    tcorr_img = tcorr_img.set({
-        'tcorr_index': TCORR_INDICES[TCORR_SOURCE],
-    })
-
     if FILL_CLIMO_FLAG:
         logging.debug('    Checking if monthly climo should be applied')
         # The climo collection names are hardcoded this way in the export scripts
@@ -250,7 +242,7 @@ def tcorr_gridded_asset_ingest(image_id, overwrite_flag=True,
             .addBands([tcorr_month_coll.first().multiply(0).rename(['quality'])])\
             .set({'tcorr_coarse_count': None})
         tcorr_img = ee.Algorithms.If(
-            ee.Number(tcorr_img.get('tcorr_coarse_count')).eq(0)
+            ee.Number(tcorr_img.get('tcorr_index')).eq(9)
                 .And(tcorr_month_coll.size().gt(0)),
             tcorr_month_img,
             tcorr_img)
@@ -293,7 +285,7 @@ def tcorr_gridded_asset_ingest(image_id, overwrite_flag=True,
             'wrs2_tile': wrs2_tile,
             'year': int(export_dt.year),
         })
-    # pprint.pprint(output_img.getInfo()['properties'])
+    # pprint.pprint(tcorr_img.getInfo()['properties'])
     # input('ENTER')
 
     logging.debug('  Building export task')
@@ -553,15 +545,23 @@ def mgrs_export_tiles(study_area_coll_id, mgrs_coll_id,
     logging.debug('Building study area collection')
     logging.debug(f'  {study_area_coll_id}')
     study_area_coll = ee.FeatureCollection(study_area_coll_id)
-    if (study_area_property == 'STUSPS' and
+    if (study_area_coll_id in ['TIGER/2018/States'] and
+            'CONUS' in [x.upper() for x in study_area_features]):
+        study_area_coll_id = 'projects/openet/featureCollections/boundaries/tiger_2018_conus'
+        study_area_coll = ee.FeatureCollection(study_area_coll_id)
+        study_area_property == 'STUSPS'
+        study_area_features = ['CONUS']
+    elif (study_area_property == 'STUSPS' and
             'CONUS' in [x.upper() for x in study_area_features]):
-        # Exclude AK, HI, AS, GU, PR, MP, VI, (but keep DC)
+        # Excluding AK, HI, AS, GU, PR, MP, VI
+        # Dropping some other states also since they aren't needed in order
+        #   to intersect all the MGRS tiles in the CONUS (CT, DE, DC, RI, VT)
         study_area_features = [
-            'AL', 'AR', 'AZ', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA',
+            'AL', 'AR', 'AZ', 'CA', 'CO', 'FL', 'GA',
             'IA', 'ID', 'IL', 'IN', 'KS', 'KY', 'LA', 'MA', 'MD', 'ME',
             'MI', 'MN', 'MO', 'MS', 'MT', 'NC', 'ND', 'NE', 'NH', 'NJ',
-            'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD',
-            'TN', 'TX', 'UT', 'VA', 'VT', 'WA', 'WI', 'WV', 'WY']
+            'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'SC', 'SD',
+            'TN', 'TX', 'UT', 'VA', 'WA', 'WI', 'WV', 'WY']
     # elif (study_area_property == 'STUSPS' and
     #         'WESTERN11' in [x.upper() for x in study_area_features]):
     #     study_area_features = [

tcorr_gridded/tcorr_gridded_assets/requirements.txt

@@ -2,4 +2,4 @@ earthengine-api>=0.1.232
 flask
 google-cloud-tasks
 openet-core>=0.0.25
-openet-ssebop>=0.1.5
+openet-ssebop>=0.1.7