diff --git a/openet/ssebop/__init__.py b/openet/ssebop/__init__.py index 5e9c771..2e81c1d 100644 --- a/openet/ssebop/__init__.py +++ b/openet/ssebop/__init__.py @@ -2,6 +2,6 @@ from .collection import Collection from . import interpolate -__version__ = "0.1.6" +__version__ = "0.1.7" MODEL_NAME = 'SSEBOP' diff --git a/openet/ssebop/image.py b/openet/ssebop/image.py index 52dafae..d835d12 100644 --- a/openet/ssebop/image.py +++ b/openet/ssebop/image.py @@ -1632,9 +1632,13 @@ def tcorr_gridded(self): quality_score_img = ee.Image([total_score_img, hotscore, coldscore, cold_rn05_score])\ .reduce(ee.Reducer.sum()) + # Set tcorr index to 9 if coarse count is 0 + # This if should be fast but the calculation below works also + tcorr_index = ee.Algorithms.If(tcorr_count_cold.gt(0), 0, 9) + return ee.Image([tcorr, quality_score_img]).rename(['tcorr', 'quality'])\ .set(self._properties)\ - .set({'tcorr_index': 0, + .set({'tcorr_index': tcorr_index, 'tcorr_coarse_count_cold': tcorr_count_cold}) @lazy_property @@ -1781,7 +1785,12 @@ def tcorr_gridded_cold(self): .reproject(crs=self.crs, crsTransform=coarse_transform)\ .updateMask(1) + # Set tcorr index to 9 if coarse count is 0 + # This "if" should be fast but the calculation approach works also + tcorr_index = ee.Algorithms.If(tcorr_count.gt(0), 1, 9) + # tcorr_index = tcorr_count.multiply(-1).max(-1).add(1).multiply(8).add(1) + return ee.Image([tcorr, total_score_img]).rename(['tcorr', 'quality']) \ .set(self._properties) \ - .set({'tcorr_index': 1, + .set({'tcorr_index': tcorr_index, 'tcorr_coarse_count': tcorr_count}) diff --git a/openet/ssebop/tests/test_c_image.py b/openet/ssebop/tests/test_c_image.py index c3fec5e..25dfa0c 100644 --- a/openet/ssebop/tests/test_c_image.py +++ b/openet/ssebop/tests/test_c_image.py @@ -1065,6 +1065,11 @@ def test_Image_tcorr_gridded_method(tcorr_source, tmax_source, image_id, 'LANDSAT/LC08/C02/T1_L2/LC08_044033_20170716', [600000, 4270000, 625000, 4285000], [612500, 4277500], [0.9914034763259638, 1]], + # TODO: Add a test where coarse count is 0 + ['GRIDDED_COLD', 'projects/usgs-ssebop/tmax/daymet_v4_mean_1981_2010_elr', + 'LANDSAT/LC08/C02/T1_L2/LC08_044033_20200910', + [600000, 4270000, 625000, 4285000], [612500, 4277500], + [None, 9]], ] ) def test_Image_tcorr_gridded_cold_method(tcorr_source, tmax_source, image_id, @@ -1080,7 +1085,10 @@ def test_Image_tcorr_gridded_cold_method(tcorr_source, tmax_source, image_id, tcorr_resample='nearest').tcorr_gridded_cold tcorr = utils.point_image_value(tcorr_img, point_xy) index = utils.getinfo(tcorr_img.get('tcorr_index')) - assert abs(tcorr['tcorr'] - expected[0]) <= tol + if expected[0] is None: + assert tcorr['tcorr'] is None + else: + assert abs(tcorr['tcorr'] - expected[0]) <= tol assert index == expected[1] diff --git a/tcorr_gridded/ini/tcorr_daymet_v4_monthly.ini b/tcorr_gridded/ini/tcorr_daymet_v4_monthly.ini index fe76f20..fe2d5ec 100644 --- a/tcorr_gridded/ini/tcorr_daymet_v4_monthly.ini +++ b/tcorr_gridded/ini/tcorr_daymet_v4_monthly.ini @@ -36,8 +36,8 @@ export_coll = projects/earthengine-legacy/assets/projects/usgs-ssebop/tcorr_grid # mgrs_tiles = 10S, 10T, 11S # utm_zones = 10, 11, 12, 13, 14, 15 -# mgrs_ftr_coll = projects/earthengine-legacy/assets/projects/openet/mgrs/conus_gridmet/zones -mgrs_ftr_coll = projects/earthengine-legacy/assets/projects/openet/mgrs/conus_gridmet/tiles +mgrs_ftr_coll = projects/earthengine-legacy/assets/projects/openet/mgrs/conus_gridmet/zones +# mgrs_ftr_coll = projects/earthengine-legacy/assets/projects/openet/mgrs/conus_gridmet/tiles [SSEBOP] diff --git a/tcorr_gridded/tcorr_export_monthly.py b/tcorr_gridded/tcorr_export_monthly.py index 15f2b16..69d2a08 100644 --- a/tcorr_gridded/tcorr_export_monthly.py +++ b/tcorr_gridded/tcorr_export_monthly.py @@ -547,15 +547,23 @@ def mgrs_export_tiles(study_area_coll_id, mgrs_coll_id, logging.debug('Building study area collection') logging.debug(f' {study_area_coll_id}') study_area_coll = ee.FeatureCollection(study_area_coll_id) - if (study_area_property == 'STUSPS' and + if (study_area_coll_id in ['TIGER/2018/States'] and 'CONUS' in [x.upper() for x in study_area_features]): - # Exclude AK, HI, AS, GU, PR, MP, VI, (but keep DC) + study_area_coll_id = 'projects/openet/featureCollections/boundaries/tiger_2018_conus' + study_area_coll = ee.FeatureCollection(study_area_coll_id) + study_area_property == 'STUSPS' + study_area_features = ['CONUS'] + elif (study_area_property == 'STUSPS' and + 'CONUS' in [x.upper() for x in study_area_features]): + # Excluding AK, HI, AS, GU, PR, MP, VI + # Dropping some other states also since they aren't needed in order + # to intersect all the MGRS tiles in the CONUS (CT, DE, DC, RI, VT) study_area_features = [ - 'AL', 'AR', 'AZ', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA', + 'AL', 'AR', 'AZ', 'CA', 'CO', 'FL', 'GA', 'IA', 'ID', 'IL', 'IN', 'KS', 'KY', 'LA', 'MA', 'MD', 'ME', 'MI', 'MN', 'MO', 'MS', 'MT', 'NC', 'ND', 'NE', 'NH', 'NJ', - 'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD', - 'TN', 'TX', 'UT', 'VA', 'VT', 'WA', 'WI', 'WV', 'WY'] + 'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'SC', 'SD', + 'TN', 'TX', 'UT', 'VA', 'WA', 'WI', 'WV', 'WY'] # elif (study_area_property == 'STUSPS' and # 'WESTERN11' in [x.upper() for x in study_area_features]): # study_area_features = [ diff --git a/tcorr_gridded/tcorr_export_scene_by_date.py b/tcorr_gridded/tcorr_export_scene_by_date.py index 002d8f1..d141285 100644 --- a/tcorr_gridded/tcorr_export_scene_by_date.py +++ b/tcorr_gridded/tcorr_export_scene_by_date.py @@ -16,7 +16,7 @@ import openet.core.utils as utils TOOL_NAME = 'tcorr_export_scene_by_date' -TOOL_VERSION = '0.1.6' +TOOL_VERSION = '0.1.7' # TODO: This could be a property or method of SSEBop or the Image class TCORR_INDICES = { @@ -567,14 +567,6 @@ def version_number(version_str): tcorr_img = t_obj.tcorr_gridded_cold # tcorr_img = t_obj.tcorr - # Properties that the climo tcorr image might change need to be set - # before the climo is used - # It would probably make more sense to move all of the property - # setting to right here instead of down below - tcorr_img = tcorr_img.set({ - 'tcorr_index': TCORR_INDICES[tcorr_source.upper()], - }) - # Replace masked tcorr images with climos # Note, If the month climo doesn't exist this will keep the # existing masked Tcorr image (we may want to change that) @@ -591,7 +583,7 @@ def version_number(version_str): .addBands([tcorr_month_coll.first().multiply(0).rename(['quality'])])\ .set({'tcorr_coarse_count': None}) tcorr_img = ee.Algorithms.If( - ee.Number(tcorr_img.get('tcorr_coarse_count')).eq(0) + ee.Number(tcorr_img.get('tcorr_index')).eq(9) .And(tcorr_month_coll.size().gt(0)), tcorr_month_img, tcorr_img) @@ -635,7 +627,7 @@ def version_number(version_str): 'wrs2_tile': wrs2_tile, 'year': int(export_dt.year), }) - # pprint.pprint(output_img.getInfo()['properties']) + # pprint.pprint(tcorr_img.getInfo()['properties']) # input('ENTER') logging.debug(' Building export task') @@ -776,15 +768,23 @@ def mgrs_export_tiles(study_area_coll_id, mgrs_coll_id, logging.debug('Building study area collection') logging.debug(f' {study_area_coll_id}') study_area_coll = ee.FeatureCollection(study_area_coll_id) - if (study_area_property == 'STUSPS' and + if (study_area_coll_id in ['TIGER/2018/States'] and + 'CONUS' in [x.upper() for x in study_area_features]): + study_area_coll_id = 'projects/openet/featureCollections/boundaries/tiger_2018_conus' + study_area_coll = ee.FeatureCollection(study_area_coll_id) + study_area_property == 'STUSPS' + study_area_features = ['CONUS'] + elif (study_area_property == 'STUSPS' and 'CONUS' in [x.upper() for x in study_area_features]): - # Exclude AK, HI, AS, GU, PR, MP, VI, (but keep DC) + # Excluding AK, HI, AS, GU, PR, MP, VI + # Dropping some other states also since they aren't needed in order + # to intersect all the MGRS tiles in the CONUS (CT, DE, DC, RI, VT) study_area_features = [ - 'AL', 'AR', 'AZ', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA', + 'AL', 'AR', 'AZ', 'CA', 'CO', 'FL', 'GA', 'IA', 'ID', 'IL', 'IN', 'KS', 'KY', 'LA', 'MA', 'MD', 'ME', 'MI', 'MN', 'MO', 'MS', 'MT', 'NC', 'ND', 'NE', 'NH', 'NJ', - 'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD', - 'TN', 'TX', 'UT', 'VA', 'VT', 'WA', 'WI', 'WV', 'WY'] + 'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'SC', 'SD', + 'TN', 'TX', 'UT', 'VA', 'WA', 'WI', 'WV', 'WY'] # elif (study_area_property == 'STUSPS' and # 'WESTERN11' in [x.upper() for x in study_area_features]): # study_area_features = [ diff --git a/tcorr_gridded/tcorr_export_scene_by_wrs2.py b/tcorr_gridded/tcorr_export_scene_by_wrs2.py index bb42b54..dc54a19 100644 --- a/tcorr_gridded/tcorr_export_scene_by_wrs2.py +++ b/tcorr_gridded/tcorr_export_scene_by_wrs2.py @@ -16,7 +16,7 @@ import openet.core.utils as utils TOOL_NAME = 'tcorr_export_scene_by_wrs2' -TOOL_VERSION = '0.1.6' +TOOL_VERSION = '0.1.7' # TODO: This could be a property or method of SSEBop or the Image class TCORR_INDICES = { @@ -571,14 +571,6 @@ def version_number(version_str): tcorr_img = t_obj.tcorr_gridded_cold # tcorr_img = t_obj.tcorr - # Properties that the climo tcorr image might change need to be set - # before the climo is used - # It would probably make more sense to move all of the property - # setting to right here instead of down below - tcorr_img = tcorr_img.set({ - 'tcorr_index': TCORR_INDICES[tcorr_source.upper()], - }) - # Replace masked tcorr images with climos # Note, If the month climo doesn't exist this will keep the # existing masked Tcorr image (we may want to change that) @@ -595,7 +587,7 @@ def version_number(version_str): .addBands([tcorr_month_coll.first().multiply(0).rename(['quality'])])\ .set({'tcorr_coarse_count': None}) tcorr_img = ee.Algorithms.If( - ee.Number(tcorr_img.get('tcorr_coarse_count')).eq(0) + ee.Number(tcorr_img.get('tcorr_index')).eq(9) .And(tcorr_month_coll.size().gt(0)), tcorr_month_img, tcorr_img) @@ -640,7 +632,7 @@ def version_number(version_str): 'wrs2_tile': wrs2_tile_fmt.format(wrs2_path, wrs2_row), 'year': int(export_dt.year), }) - # pprint.pprint(output_img.getInfo()['properties']) + # pprint.pprint(tcorr_img.getInfo()['properties']) # input('ENTER') logging.debug(' Building export task') @@ -782,15 +774,23 @@ def mgrs_export_tiles(study_area_coll_id, mgrs_coll_id, logging.debug('Building study area collection') logging.debug(f' {study_area_coll_id}') study_area_coll = ee.FeatureCollection(study_area_coll_id) - if (study_area_property == 'STUSPS' and + if (study_area_coll_id in ['TIGER/2018/States'] and + 'CONUS' in [x.upper() for x in study_area_features]): + study_area_coll_id = 'projects/openet/featureCollections/boundaries/tiger_2018_conus' + study_area_coll = ee.FeatureCollection(study_area_coll_id) + study_area_property == 'STUSPS' + study_area_features = ['CONUS'] + elif (study_area_property == 'STUSPS' and 'CONUS' in [x.upper() for x in study_area_features]): - # Exclude AK, HI, AS, GU, PR, MP, VI, (but keep DC) + # Excluding AK, HI, AS, GU, PR, MP, VI + # Dropping some other states also since they aren't needed in order + # to intersect all the MGRS tiles in the CONUS (CT, DE, DC, RI, VT) study_area_features = [ - 'AL', 'AR', 'AZ', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA', + 'AL', 'AR', 'AZ', 'CA', 'CO', 'FL', 'GA', 'IA', 'ID', 'IL', 'IN', 'KS', 'KY', 'LA', 'MA', 'MD', 'ME', 'MI', 'MN', 'MO', 'MS', 'MT', 'NC', 'ND', 'NE', 'NH', 'NJ', - 'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD', - 'TN', 'TX', 'UT', 'VA', 'VT', 'WA', 'WI', 'WV', 'WY'] + 'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'SC', 'SD', + 'TN', 'TX', 'UT', 'VA', 'WA', 'WI', 'WV', 'WY'] # elif (study_area_property == 'STUSPS' and # 'WESTERN11' in [x.upper() for x in study_area_features]): # study_area_features = [ @@ -804,7 +804,7 @@ def mgrs_export_tiles(study_area_coll_id, mgrs_coll_id, study_area_coll = study_area_coll.filter( ee.Filter.inList(study_area_property, study_area_features)) - logging.info('Building MGRS tile list') + logging.debug('Building MGRS tile list') tiles_coll = ee.FeatureCollection(mgrs_coll_id) \ .filterBounds(study_area_coll.geometry()) diff --git a/tcorr_gridded/tcorr_gridded_assets/main.py b/tcorr_gridded/tcorr_gridded_assets/main.py index e701390..87e74b3 100644 --- a/tcorr_gridded/tcorr_gridded_assets/main.py +++ b/tcorr_gridded/tcorr_gridded_assets/main.py @@ -16,7 +16,7 @@ logging.getLogger('googleapiclient').setLevel(logging.ERROR) TOOL_NAME = 'tcorr_gridded_cloud_function' -TOOL_VERSION = '0.1.0' +TOOL_VERSION = '0.1.7' # TODO: Move all of these to config.py? FUNCTION_URL = 'https://us-central1-ssebop.cloudfunctions.net' @@ -228,14 +228,6 @@ def tcorr_gridded_asset_ingest(image_id, overwrite_flag=True, tcorr_img = t_obj.tcorr_gridded_cold # tcorr_img = t_obj.tcorr - # Properties that the climo tcorr image might change need to be set - # before the climo is used - # It would probably make more sense to move all of the property - # setting to right here instead of down below - tcorr_img = tcorr_img.set({ - 'tcorr_index': TCORR_INDICES[TCORR_SOURCE], - }) - if FILL_CLIMO_FLAG: logging.debug(' Checking if monthly climo should be applied') # The climo collection names are hardcoded this way in the export scripts @@ -250,7 +242,7 @@ def tcorr_gridded_asset_ingest(image_id, overwrite_flag=True, .addBands([tcorr_month_coll.first().multiply(0).rename(['quality'])])\ .set({'tcorr_coarse_count': None}) tcorr_img = ee.Algorithms.If( - ee.Number(tcorr_img.get('tcorr_coarse_count')).eq(0) + ee.Number(tcorr_img.get('tcorr_index')).eq(9) .And(tcorr_month_coll.size().gt(0)), tcorr_month_img, tcorr_img) @@ -293,7 +285,7 @@ def tcorr_gridded_asset_ingest(image_id, overwrite_flag=True, 'wrs2_tile': wrs2_tile, 'year': int(export_dt.year), }) - # pprint.pprint(output_img.getInfo()['properties']) + # pprint.pprint(tcorr_img.getInfo()['properties']) # input('ENTER') logging.debug(' Building export task') @@ -553,15 +545,23 @@ def mgrs_export_tiles(study_area_coll_id, mgrs_coll_id, logging.debug('Building study area collection') logging.debug(f' {study_area_coll_id}') study_area_coll = ee.FeatureCollection(study_area_coll_id) - if (study_area_property == 'STUSPS' and + if (study_area_coll_id in ['TIGER/2018/States'] and + 'CONUS' in [x.upper() for x in study_area_features]): + study_area_coll_id = 'projects/openet/featureCollections/boundaries/tiger_2018_conus' + study_area_coll = ee.FeatureCollection(study_area_coll_id) + study_area_property == 'STUSPS' + study_area_features = ['CONUS'] + elif (study_area_property == 'STUSPS' and 'CONUS' in [x.upper() for x in study_area_features]): - # Exclude AK, HI, AS, GU, PR, MP, VI, (but keep DC) + # Excluding AK, HI, AS, GU, PR, MP, VI + # Dropping some other states also since they aren't needed in order + # to intersect all the MGRS tiles in the CONUS (CT, DE, DC, RI, VT) study_area_features = [ - 'AL', 'AR', 'AZ', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA', + 'AL', 'AR', 'AZ', 'CA', 'CO', 'FL', 'GA', 'IA', 'ID', 'IL', 'IN', 'KS', 'KY', 'LA', 'MA', 'MD', 'ME', 'MI', 'MN', 'MO', 'MS', 'MT', 'NC', 'ND', 'NE', 'NH', 'NJ', - 'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD', - 'TN', 'TX', 'UT', 'VA', 'VT', 'WA', 'WI', 'WV', 'WY'] + 'NM', 'NV', 'NY', 'OH', 'OK', 'OR', 'PA', 'SC', 'SD', + 'TN', 'TX', 'UT', 'VA', 'WA', 'WI', 'WV', 'WY'] # elif (study_area_property == 'STUSPS' and # 'WESTERN11' in [x.upper() for x in study_area_features]): # study_area_features = [ diff --git a/tcorr_gridded/tcorr_gridded_assets/requirements.txt b/tcorr_gridded/tcorr_gridded_assets/requirements.txt index 658c2db..412a0b5 100644 --- a/tcorr_gridded/tcorr_gridded_assets/requirements.txt +++ b/tcorr_gridded/tcorr_gridded_assets/requirements.txt @@ -2,4 +2,4 @@ earthengine-api>=0.1.232 flask google-cloud-tasks openet-core>=0.0.25 -openet-ssebop>=0.1.5 \ No newline at end of file +openet-ssebop>=0.1.7 \ No newline at end of file