readme fix

allow config override via override parameter
fix ee session initialization
config now uses externally provided .env file to setup
added notebook examples from the Applied Deep Learning Book Chapter 1

README.md

@@ -23,6 +23,7 @@ from aces.config import Config
 from aces.model_trainer import ModelTrainer
 
 if __name__ == "__main__":
+    config_file = "config.env"
     config = Config()
     trainer = ModelTrainer(config)
     trainer.train_model()

aces/config.py

@@ -95,9 +95,9 @@ class Config:
         GCP_MACHINE_TYPE (str): The Google Cloud Platform machine type.
     """
 
-    def __init__(self, config_file) -> None:
+    def __init__(self, config_file, override=False) -> None:
 
-        load_dotenv(config_file)
+        load_dotenv(config_file, override=override)
 
         self.BASEDIR = Path(os.getenv("BASEDIR"))
         _DATADIR = os.getenv("DATADIR")

aces/ee_utils.py

@@ -36,7 +36,7 @@ def get_credentials_by_service_account_key(key):
         return credentials
 
     @staticmethod
-    def initialize_session(use_highvolume : bool = False, key : Union[str, None] = None):
+    def initialize_session(use_highvolume : bool = False, key : Union[str, None] = None, project: str = None):
         """
         Initialize the Earth Engine session.
 
@@ -45,14 +45,22 @@ def initialize_session(use_highvolume : bool = False, key : Union[str, None] = N
         key (str or None): The path to the service account key JSON file. If None, the default credentials will be used.
         """
         if key is None:
-            if use_highvolume:
+            if use_highvolume and project:
+                ee.Initialize(opt_url="https://earthengine-highvolume.googleapis.com", project=project)
+            elif use_highvolume:
                 ee.Initialize(opt_url="https://earthengine-highvolume.googleapis.com")
+            elif project:
+                ee.Initialize(project=project)
             else:
                 ee.Initialize()
         else:
             credentials = EEUtils.get_credentials_by_service_account_key(key)
-            if use_highvolume:
+            if use_highvolume and project:
+                ee.Initialize(credentials, opt_url="https://earthengine-highvolume.googleapis.com", project=project)
+            elif use_highvolume:
                 ee.Initialize(credentials, opt_url="https://earthengine-highvolume.googleapis.com")
+            elif project:
+                ee.Initialize(credentials, project=project)
             else:
                 ee.Initialize(credentials)
 

workflow/v2/4.export_image_for_prediction.py

@@ -78,47 +78,50 @@
 composite_during = composite_during.regexpRename("$(.*)", "_during")
 image = composite_before.addBands(composite_during).toFloat()
 
-if Config.USE_ELEVATION:
+config_file = "config.env"
+config = Config(config_file)
+
+if config.USE_ELEVATION:
     elevation = ee.Image("projects/servir-sco-assets/assets/Bhutan/ACES_2/elevationParo")
     slope = ee.Image("projects/servir-sco-assets/assets/Bhutan/ACES_2/slopeParo")
     image = image.addBands(elevation).addBands(slope).toFloat()
-    Config.FEATURES.extend(["elevation", "slope"])
+    config.FEATURES.extend(["elevation", "slope"])
 
 
-if Config.USE_S1:
+if config.USE_S1:
     sentinel1_asc_before_composite = ee.Image("projects/servir-sco-assets/assets/Bhutan/Sentinel1Ascending2021/s1AscBefore")
     sentinel1_asc_during_composite = ee.Image("projects/servir-sco-assets/assets/Bhutan/Sentinel1Ascending2021/s1AscDuring")
     sentinel1_desc_before_composite = ee.Image("projects/servir-sco-assets/assets/Bhutan/Sentinel1Descending2021/s1DescBefore")
     sentinel1_desc_during_composite = ee.Image("projects/servir-sco-assets/assets/Bhutan/Sentinel1Descending2021/s1DescDuring")
 
     image = image.addBands(sentinel1_asc_before_composite).addBands(sentinel1_asc_during_composite).addBands(sentinel1_desc_before_composite).addBands(sentinel1_desc_during_composite).toFloat()
-    Config.FEATURES.extend(["vv_asc_before", "vh_asc_before", "vv_asc_during", "vh_asc_during",
+    config.FEATURES.extend(["vv_asc_before", "vh_asc_before", "vv_asc_during", "vh_asc_during",
                             "vv_desc_before", "vh_desc_before", "vv_desc_during", "vh_desc_during"])
 
 # dem = ee.Image("MERIT/DEM/v1_0_3") # ee.Image('USGS/SRTMGL1_003');
 # dem = dem.clip(fc_country)
 # riceZone = dem.gt(rice_zone[region]["min"]).And(dem.lte(rice_zone[region]["max"]))
 # image = image.clip(region_fc).updateMask(riceZone)
 
-image = image.select(Config.FEATURES)
+image = image.select(config.FEATURES)
 print("image", image.bandNames().getInfo())
 
 # Specify patch and file dimensions.
 formatOptions = {
-  "patchDimensions": [Config.PATCH_SHAPE_SINGLE, Config.PATCH_SHAPE_SINGLE],
+  "patchDimensions": [config.PATCH_SHAPE_SINGLE, config.PATCH_SHAPE_SINGLE],
   "maxFileSize": 104857600,
   "compressed": True
 }
 
-if Config.KERNEL_BUFFER:
-    formatOptions["kernelSize"] = Config.KERNEL_BUFFER
+if config.KERNEL_BUFFER:
+    formatOptions["kernelSize"] = config.KERNEL_BUFFER
 
 # Setup the task
 image_export_options = {
-    "description": Config.GCS_IMAGE_DIR.split("/")[-1],
-    "file_name_prefix": f"{Config.GCS_IMAGE_DIR}/{Config.GCS_IMAGE_PREFIX}",
-    "bucket": Config.GCS_BUCKET,
-    "scale": Config.SCALE,
+    "description": "export_task_for_prediction",
+    "file_name_prefix": f"{config.GCS_IMAGE_DIR}/{config.GCS_IMAGE_PREFIX}",
+    "bucket": config.GCS_BUCKET,
+    "scale": config.SCALE,
     "file_format": "TFRecord",
     "region": region_fc, # image.geometry(),
     "format_options": formatOptions,

workflow/v2/5.prediction_unet.py

@@ -16,21 +16,24 @@
 import subprocess
 
 
-OUTPUT_IMAGE_FILE = str(Config.MODEL_DIR / "prediction" / f"{Config.OUTPUT_NAME}.TFRecord")
-if not os.path.exists(str(Config.MODEL_DIR / "prediction")): os.mkdir(str(Config.MODEL_DIR / "prediction"))
+config_file = "config.env"
+config = Config(config_file)
+
+OUTPUT_IMAGE_FILE = str(config.MODEL_DIR / "prediction" / f"{config.OUTPUT_NAME}.TFRecord")
+if not os.path.exists(str(config.MODEL_DIR / "prediction")): os.mkdir(str(config.MODEL_DIR / "prediction"))
 print(f"OUTPUT_IMAGE_FILE: {OUTPUT_IMAGE_FILE}")
 
-OUTPUT_GCS_PATH = f"gs://{Config.GCS_BUCKET}/prediction/{Config.OUTPUT_NAME}.TFRecord"
+OUTPUT_GCS_PATH = f"gs://{config.GCS_BUCKET}/prediction/{config.OUTPUT_NAME}.TFRecord"
 print(f"OUTPUT_GCS_PATH: {OUTPUT_GCS_PATH}")
 
-ls = f"sudo gsutil ls gs://{Config.GCS_BUCKET}/{Config.GCS_IMAGE_DIR}/"
+ls = f"sudo gsutil ls gs://{config.GCS_BUCKET}/{config.GCS_IMAGE_DIR}/"
 print(f"ls >> : {ls}")
 files_list = subprocess.check_output(ls, shell=True)
 files_list = files_list.decode("utf-8")
 files_list = files_list.split("\n")
 
 # Get only the files generated by the image export.
-exported_files_list = [s for s in files_list if Config.GCS_IMAGE_PREFIX in s]
+exported_files_list = [s for s in files_list if config.GCS_IMAGE_PREFIX in s]
 
 print(f"exported_files_list: {exported_files_list}")
 
@@ -51,11 +54,11 @@
 print(f"json_file: {json_file}")
 
 if Config.USE_BEST_MODEL_FOR_INFERENCE:
-    print(f"Using best model for inference.\nLoading model from {str(Config.MODEL_DIR)}/{Config.MODEL_CHECKPOINT_NAME}.tf")
-    this_model = tf.keras.models.load_model(f"{str(Config.MODEL_DIR)}/{Config.MODEL_CHECKPOINT_NAME}.tf")
+    print(f"Using best model for inference.\nLoading model from {str(config.MODEL_DIR)}/{config.MODEL_CHECKPOINT_NAME}.tf")
+    this_model = tf.keras.models.load_model(f"{str(config.MODEL_DIR)}/{config.MODEL_CHECKPOINT_NAME}.tf")
 else:
-    print(f"Using last model for inference.\nLoading model from {str(Config.MODEL_DIR)}/trained-model")
-    this_model = tf.keras.models.load_model(f"{str(Config.MODEL_DIR)}/trained-model")
+    print(f"Using last model for inference.\nLoading model from {str(config.MODEL_DIR)}/trained-model")
+    this_model = tf.keras.models.load_model(f"{str(config.MODEL_DIR)}/trained-model")
 
 print(this_model.summary())
 
@@ -74,40 +77,40 @@
 patch_dimensions_flat = [patch_width * patch_height, 1]
 
 # Get set up for prediction.
-if Config.KERNEL_BUFFER:
-    x_buffer = Config.KERNEL_BUFFER[0] // 2
-    y_buffer = Config.KERNEL_BUFFER[1] // 2
+if config.KERNEL_BUFFER:
+    x_buffer = config.KERNEL_BUFFER[0] // 2
+    y_buffer = config.KERNEL_BUFFER[1] // 2
 
     buffered_shape = [
-        Config.PATCH_SHAPE[0] + Config.KERNEL_BUFFER[0],
-        Config.PATCH_SHAPE[1] + Config.KERNEL_BUFFER[1],
+        config.PATCH_SHAPE[0] + config.KERNEL_BUFFER[0],
+        config.PATCH_SHAPE[1] + config.KERNEL_BUFFER[1],
     ]
 else:
     x_buffer = 0
     y_buffer = 0
-    buffered_shape = Config.PATCH_SHAPE
+    buffered_shape = config.PATCH_SHAPE
 
-if Config.USE_ELEVATION:
-    Config.FEATURES.extend(["elevation", "slope"])
+if config.USE_ELEVATION:
+    config.FEATURES.extend(["elevation", "slope"])
 
 
-if Config.USE_S1:
-    Config.FEATURES.extend(["vv_asc_before", "vh_asc_before", "vv_asc_during", "vh_asc_during",
+if config.USE_S1:
+    config.FEATURES.extend(["vv_asc_before", "vh_asc_before", "vv_asc_during", "vh_asc_during",
                             "vv_desc_before", "vh_desc_before", "vv_desc_during", "vh_desc_during"])
 
-print(f"Config.FEATURES: {Config.FEATURES}")
+print(f"Config.FEATURES: {config.FEATURES}")
 
 image_columns = [
-    tf.io.FixedLenFeature(shape=buffered_shape, dtype=tf.float32) for k in Config.FEATURES
+    tf.io.FixedLenFeature(shape=buffered_shape, dtype=tf.float32) for k in config.FEATURES
 ]
 
-image_features_dict = dict(zip(Config.FEATURES, image_columns))
+image_features_dict = dict(zip(config.FEATURES, image_columns))
 
 def parse_image(example_proto):
     return tf.io.parse_single_example(example_proto, image_features_dict)
 
 def toTupleImage(inputs):
-    inputsList = [inputs.get(key) for key in Config.FEATURES]
+    inputsList = [inputs.get(key) for key in config.FEATURES]
     stacked = tf.stack(inputsList, axis=0)
     stacked = tf.transpose(stacked, [1, 2, 0])
     return stacked
@@ -139,8 +142,8 @@ def toTupleImage(inputs):
         print(f"Writing patch {i}...")
 
     prediction_patch = prediction_patch[
-        x_buffer: x_buffer+Config.PATCH_SHAPE[0],
-        y_buffer: y_buffer+Config.PATCH_SHAPE[1]
+        x_buffer: x_buffer+config.PATCH_SHAPE[0],
+        y_buffer: y_buffer+config.PATCH_SHAPE[1]
     ]
 
     example = tf.train.Example(
@@ -181,6 +184,6 @@ def toTupleImage(inputs):
 print(f"uploading classified image to earth engine: {result}")
 
 # upload to earth engine asset
-upload_image = f"earthengine upload image --asset_id={Config.EE_OUTPUT_ASSET}/{Config.OUTPUT_NAME} --pyramiding_policy=mode {OUTPUT_GCS_PATH} {json_file}"
+upload_image = f"earthengine upload image --asset_id={config.EE_OUTPUT_ASSET}/{config.OUTPUT_NAME} --pyramiding_policy=mode {OUTPUT_GCS_PATH} {json_file}"
 result = subprocess.check_output(upload_image, shell=True)
 print(f"uploading classified image to earth engine: {result}")