address issue #46 and #47

improve README
bugfixes associated with it

.env.example

@@ -40,24 +40,6 @@ nir_during"
 USE_ELEVATION = False
 USE_S1 = False
 
-DERIVE_FEATURES = False
-ADDED_FEATURES = "ndvi_before
-ndvi_during
-evi_before
-evi_during
-ndwi_before
-ndwi_during
-savi_before
-savi_during
-msavi_before
-msavi_during
-mtvi2_before
-mtvi2_during
-vari_before
-vari_during
-tgi_before
-tgi_during"
-
 # Multiple labels should be multiline as FEATURES
 LABELS = "class"
 
@@ -108,10 +90,10 @@ USE_AI_PLATFORM = False
 # dnn does not do augmentation
 TRANSFORM_DATA = True
 
-# ACTIVATION_FN autogenerated in the main script
-# if n_class = 1: activation_fn = "sigmoid"
-# else: activation_fn = "softmax"
+# checks if activation_fn = "sigmoid" then n_class = 1
+# checks if activation_fn = "softmax" then n_class > 1
 ACTIVATION_FN = "softmax"
+
 OPTIMIZER = "adam"
 # standard or custom available
 # current custom available are: "custom_focal_tversky_loss"

aces/config.py

@@ -10,6 +10,8 @@
 import ast
 from dotenv import load_dotenv
 
+from aces.utils import Utils
+
 
 class Config:
     """
@@ -30,8 +32,6 @@ class Config:
         FEATURES (str): The list of features used in the model.
         USE_ELEVATION (bool): Flag to use elevation data.
         USE_S1 (bool): Flag to use Sentinel-1 data.
-        DERIVE_FEATURES (bool): Flag to derive features.
-        ADDED_FEATURES (list): A list of additional features used in the model (used when DERIVE_FEATURES is true).
         LABELS (list): A list of labels used in the model.
         SCALE (int): The scale of the data.
         # Note: The seeding componenet needs fixing. It is not working as expected.
@@ -117,31 +117,43 @@ def __init__(self, config_file, override=False) -> None:
             self.DATADIR = self.BASEDIR / os.getenv("DATADIR")
             self.TRAINING_DIR = self.DATADIR / "training"
             self.TESTING_DIR = self.DATADIR / "testing"
-            self.VALIDATION_DIR= self.DATADIR / "validation"
+            self.VALIDATION_DIR = self.DATADIR / "validation"
 
         print(f"BASEDIR: {self.BASEDIR}")
-        print(f"DATADIR: {self.DATADIR}")
+        print(f"checking if the DATADIR exists at: {self.DATADIR}")
+        if not Path(f"{self.DATADIR}").is_dir():
+            raise FileNotFoundError(f"The directory '{self.DATADIR}' does not exist.")
 
         self.OUTPUT_DIR = self.BASEDIR / os.getenv("OUTPUT_DIR")
 
         self.MODEL_NAME = os.getenv("MODEL_NAME")
+        if self.MODEL_NAME is None:
+            self.MODEL_NAME = "aces_model"
+
         self.MODEL_CHECKPOINT_NAME = os.getenv("MODEL_CHECKPOINT_NAME")
 
         self.MODEL_DIR_NAME = os.getenv("MODEL_DIR_NAME")
 
         self.MODEL_DIR = self.OUTPUT_DIR / self.MODEL_DIR_NAME
+        # Ensure MODEL_DIR directory exists, create if it doesn't
+        # this takes care of creating OUTPUT_DIR as well
+        print(f"creating if MODEL_DIR does not exist at: {self.MODEL_DIR}")
+        self.MODEL_DIR.mkdir(parents=True, exist_ok=True)
 
         self.AUTO_MODEL_DIR_NAME = os.getenv("AUTO_MODEL_DIR_NAME") == "True"
 
         self.DATA_OUTPUT_DIR = os.getenv("DATA_OUTPUT_DIR")
 
         self.SCALE = int(os.getenv("SCALE"))
 
-        self.FEATURES = os.getenv("FEATURES").split("\n")
-        self.ADDED_FEATURES = os.getenv("ADDED_FEATURES").split("\n")
+        # self.FEATURES = os.getenv("FEATURES").split("\n")
+        self.FEATURES = Utils.parse_params("FEATURES")
+
         self.USE_ELEVATION = os.getenv("USE_ELEVATION") == "True"
         self.USE_S1 = os.getenv("USE_S1") == "True"
-        self.LABELS = os.getenv("LABELS").split("\n")
+
+        # self.LABELS = os.getenv("LABELS").split("\n")
+        self.LABELS = Utils.parse_params("LABELS")
 
         if self.USE_ELEVATION:
                 self.FEATURES.extend(["elevation", "slope"])
@@ -154,15 +166,20 @@ def __init__(self, config_file, override=False) -> None:
         print(f"using labels: {self.LABELS}")
 
         self.USE_SEED = os.getenv("USE_SEED") == "True"
-        self.SEED = int(os.getenv("SEED"))
+        if self.USE_SEED:
+            self.SEED = int(os.getenv("SEED"))
 
-        self.PRINT_INFO = os.getenv("USE_SEED") == "True"
+        self.PRINT_INFO = os.getenv("PRINT_INFO")
+        if self.PRINT_INFO is None:
+            self.PRINT_INFO = True
+        else:
+            self.PRINT_INFO = self.PRINT_INFO == "True"
 
         # patch size for training
         self.PATCH_SHAPE = ast.literal_eval(os.getenv("PATCH_SHAPE"))
         self.PATCH_SHAPE_SINGLE = self.PATCH_SHAPE[0]
         KERNEL_BUFFER = os.getenv("KERNEL_BUFFER")
-        if KERNEL_BUFFER == "0":
+        if KERNEL_BUFFER == "0" or KERNEL_BUFFER is None:
             self.KERNEL_BUFFER = None
         else:
             self.KERNEL_BUFFER = ast.literal_eval(KERNEL_BUFFER)
@@ -177,14 +194,39 @@ def __init__(self, config_file, override=False) -> None:
 
         self.BATCH_SIZE = int(os.getenv("BATCH_SIZE"))
         self.EPOCHS = int(os.getenv("EPOCHS"))
-        self.RAMPUP_EPOCHS = int(os.getenv("RAMPUP_EPOCHS"))
-        self.SUSTAIN_EPOCHS = int(os.getenv("SUSTAIN_EPOCHS"))
+
+        self.RAMPUP_EPOCHS = os.getenv("RAMPUP_EPOCHS")
+        if self.RAMPUP_EPOCHS is not None:
+            self.RAMPUP_EPOCHS = int(self.RAMPUP_EPOCHS)
+
+        self.SUSTAIN_EPOCHS = os.getenv("SUSTAIN_EPOCHS")
+        if self.RAMPUP_EPOCHS is not None and self.SUSTAIN_EPOCHS is None:
+            raise ValueError("SUSTAIN_EPOCHS must be set if RAMPUP_EPOCHS is set.")
+
+        if self.SUSTAIN_EPOCHS is not None and self.RAMPUP_EPOCHS is not None:
+            self.SUSTAIN_EPOCHS = int(self.SUSTAIN_EPOCHS)
 
         self.USE_ADJUSTED_LR = os.getenv("USE_ADJUSTED_LR") == "True"
-        self.MAX_LR = float(os.getenv("MAX_LR"))
-        self.MID_LR = float(os.getenv("MID_LR"))
-        self.MIN_LR = float(os.getenv("MIN_LR"))
-        self.DROPOUT_RATE = float(os.getenv("DROPOUT_RATE"))
+        if self.USE_ADJUSTED_LR and not self.RAMPUP_EPOCHS and not self.SUSTAIN_EPOCHS:
+            raise ValueError("RAMPUP_EPOCHS and SUSTAIN_EPOCHS must be set if USE_ADJUSTED_LR is set.")
+
+        if self.USE_ADJUSTED_LR:
+            try:
+                self.MAX_LR = float(os.getenv("MAX_LR"))
+                self.MID_LR = float(os.getenv("MID_LR"))
+                self.MIN_LR = float(os.getenv("MIN_LR"))
+            except ValueError:
+                raise ValueError("MAX_LR, MID_LR, and MIN_LR must be set if USE_ADJUSTED_LR is set.")
+
+        self.DROPOUT_RATE = os.getenv("DROPOUT_RATE")
+        if self.DROPOUT_RATE is not None:
+            self.DROPOUT_RATE = float(self.DROPOUT_RATE)
+            if self.DROPOUT_RATE < 0.:
+                self.DROPOUT_RATE = 0.
+            elif self.DROPOUT_RATE > 1.:
+                self.DROPOUT_RATE = 1.
+        else:
+            self.DROPOUT_RATE = 0.
 
         self.LOSS = os.getenv("LOSS")
 
@@ -194,12 +236,28 @@ def __init__(self, config_file, override=False) -> None:
 
         self.USE_BEST_MODEL_FOR_INFERENCE = os.getenv("USE_BEST_MODEL_FOR_INFERENCE") == "True"
 
-        self.ACTIVATION_FN = "sigmoid" if self.OUT_CLASS_NUM == 1 else "softmax"
+        self.ACTIVATION_FN = os.getenv("ACTIVATION_FN")
+        if self.ACTIVATION_FN == "sigmoid" and self.OUT_CLASS_NUM > 1:
+            raise ValueError("Sigmoid activation function is only for binary classification.")
+        if self.ACTIVATION_FN == "softmax" and self.OUT_CLASS_NUM == 1:
+            raise ValueError("Softmax activation function is only for multi-class classification.")
+
         self.CALLBACK_PARAMETER = os.getenv("CALLBACK_PARAMETER")
+        if self.CALLBACK_PARAMETER is None:
+            self.CALLBACK_PARAMETER = "val_loss"
 
         self.EARLY_STOPPING = os.getenv("EARLY_STOPPING") == "True"
-        self.TRANSFORM_DATA = os.getenv("TRANSFORM_DATA") == "True"
-        self.DERIVE_FEATURES = os.getenv("DERIVE_FEATURES") == "True"
+
+        # self.TRANSFORM_DATA = os.getenv("TRANSFORM_DATA") == "True"
+        self.TRANSFORM_DATA = os.getenv("TRANSFORM_DATA")
+        if self.TRANSFORM_DATA is None:
+            self.TRANSFORM_DATA = True
+        else:
+            self.TRANSFORM_DATA = self.TRANSFORM_DATA == "True"
+
+        # DERIVE_FEATURES & ADDED_FEATURES are currently not available
+        # The original idea was if DERVIRE_FEATURES is True, then the ADDED_FEATURES are concatenated
+        # This is not fully implemented yet
 
         # EE settings
         self.USE_SERVICE_ACCOUNT = os.getenv("USE_SERVICE_ACCOUNT") == "True"

aces/model_builder.py

@@ -111,13 +111,13 @@ def build_and_compile_dnn_model(self, **kwargs):
         inputs = keras.Input(shape=(None, self.in_size), name="input_layer")
 
         x = keras.layers.Dense(256, activation="relu")(inputs)
-        x = keras.layers.Dropout(0.2)(x)
+        x = keras.layers.Dropout(kwargs.get("DROPOUT_RATE", 0.))(x)
         x = keras.layers.Dense(128, activation="relu")(x)
-        x = keras.layers.Dropout(0.2)(x)
+        x = keras.layers.Dropout(kwargs.get("DROPOUT_RATE", 0.))(x)
         x = keras.layers.Dense(64, activation="relu")(x)
-        x = keras.layers.Dropout(0.2)(x)
+        x = keras.layers.Dropout(kwargs.get("DROPOUT_RATE", 0.))(x)
         x = keras.layers.Dense(32, activation="relu")(x)
-        x = keras.layers.Dropout(0.2)(x)
+        x = keras.layers.Dropout(kwargs.get("DROPOUT_RATE", 0.))(x)
         output = keras.layers.Dense(self.out_classes, activation=kwargs.get("ACTIVATION_FN"), bias_initializer=INITIAL_BIAS)(x)
 
         model = keras.models.Model(inputs=inputs, outputs=output)
@@ -161,13 +161,13 @@ def build_and_compile_dnn_model_for_ai_platform(self, **kwargs):
             inputs = inputs_main
 
         x = keras.layers.Conv2D(256, (1, 1), activation="relu")(inputs)
-        x = keras.layers.Dropout(0.2)(x)
+        x = keras.layers.Dropout(kwargs.get("DROPOUT_RATE"), 0.)(x)
         x = keras.layers.Conv2D(128, (1, 1), activation="relu")(x)
-        x = keras.layers.Dropout(0.2)(x)
+        x = keras.layers.Dropout(kwargs.get("DROPOUT_RATE"), 0.)(x)
         x = keras.layers.Conv2D(64, (1, 1), activation="relu")(x)
-        x = keras.layers.Dropout(0.2)(x)
+        x = keras.layers.Dropout(kwargs.get("DROPOUT_RATE"), 0.)(x)
         x = keras.layers.Conv2D(32, (1, 1), activation="relu")(x)
-        x = keras.layers.Dropout(0.2)(x)
+        x = keras.layers.Dropout(kwargs.get("DROPOUT_RATE"), 0.)(x)
         output = keras.layers.Conv2D(self.out_classes, (1, 1), activation=kwargs.get("ACTIVATION_FN"), bias_initializer=INITIAL_BIAS)(x)
 
         model = keras.models.Model(inputs=inputs_main, outputs=output)
@@ -196,16 +196,16 @@ def build_and_compile_cnn_model(self, **kwargs):
         inputs = keras.Input(shape=(kwargs.get("PATCH_SHAPE", 128)[0], kwargs.get("PATCH_SHAPE", 128)[0], self.in_size))
         x = keras.layers.Conv2D(32, 3, activation="relu", name="convd-1", padding="same")(inputs)
         x = keras.layers.BatchNormalization()(x)
-        x = keras.layers.Dropout(0.15)(x)
+        x = keras.layers.Dropout(kwargs.get("DROPOUT_RATE"), 0.)(x)
         x = keras.layers.Conv2D(64, 3, activation="relu", name="convd-2", padding="same")(x)
         x = keras.layers.BatchNormalization()(x)
-        x = keras.layers.Dropout(0.15)(x)
+        x = keras.layers.Dropout(kwargs.get("DROPOUT_RATE"), 0.)(x)
         x = keras.layers.Conv2D(128, 3, activation="relu", name="convd-3", padding="same")(x)
         x = keras.layers.BatchNormalization()(x)
-        x = keras.layers.Dropout(0.15)(x)
+        x = keras.layers.Dropout(kwargs.get("DROPOUT_RATE"), 0.)(x)
         x = keras.layers.Conv2D(128, 3, activation="relu", name="convd-4", padding="same")(x)
         x = keras.layers.BatchNormalization()(x)
-        x = keras.layers.Dropout(0.15)(x)
+        x = keras.layers.Dropout(kwargs.get("DROPOUT_RATE"), 0.)(x)
         outputs = keras.layers.Conv2D(self.out_classes, (1, 1), activation="softmax", name="final_conv")(x)
         model = keras.Model(inputs, outputs, name="cnn_model")
 
@@ -298,8 +298,6 @@ def _build_and_compile_unet_model(self, **kwargs):
 
         DERIVE_FEATURES = kwargs.get("DERIVE_FEATURES", False)
 
-        print(f"DERIVE_FEATURES: {DERIVE_FEATURES}")
-
         if DERIVE_FEATURES:
             input_features = rs.concatenate_features_for_cnn(inputs)
         else:

aces/model_trainer.py

@@ -13,7 +13,7 @@
 
 from aces.config import Config
 from aces.metrics import Metrics
-from aces.model_builder import ModelBuilder, DeSerializeInput, ReSerializeOutput, AddExtraFeatures
+from aces.model_builder import ModelBuilder, DeSerializeInput, ReSerializeOutput
 from aces.data_processor import DataProcessor
 from aces.utils import Utils, TFUtils
 
@@ -27,31 +27,27 @@ class ModelTrainer:
         model_builder: An instance of ModelBuilder for building the model.
         build_model: A partial function for building the model with the specified model type.
     """
-    def __init__(self, config: Config, use_seed: bool = True, seed: int = 42):
+    def __init__(self, config: Config):
         """
         Initialize the ModelTrainer object.
 
         Args:
             config: An object containing the configuration settings for model training.
-            use_seed: A flag indicating whether to use a seed for reproducibility (needs fixing).
-            seed: The seed value to use for reproducibility.
 
         Attributes:
             config: The configuration settings for model training.
             model_builder: An instance of ModelBuilder for building the model.
             build_model: A partial function for building the model with the specified model type.
         """
         self.config = config
-        self.use_seed = use_seed
-        self.seed = seed
         # @FIXME: This isn't producing reproducable results
-        if self.use_seed:
+        if self.config.USE_SEED:
             # producable results
             import random
-            print(f"Using seed: {self.seed}")
-            tf.random.set_seed(self.seed)
-            np.random.seed(self.seed)
-            random.seed(2)
+            print(f"Using seed: {self.config.SEED}")
+            tf.random.set_seed(self.config.SEED)
+            np.random.seed(self.config.SEED)
+            random.seed(self.config.SEED)
 
         # @ToDO: Create a way to autoload loss function from the list without if else
         if self.config.LOSS == "custom_focal_tversky_loss":
@@ -68,7 +64,7 @@ def __init__(self, config: Config, use_seed: bool = True, seed: int = 42):
         )
         self.build_model = partial(self.model_builder.build_model, model_type=self.config.MODEL_TYPE,
                                    **{"FOR_AI_PLATFORM": self.config.USE_AI_PLATFORM,
-                                      "DERIVE_FEATURES": self.config.DERIVE_FEATURES})
+                                      "DERIVE_FEATURES": self.config.DERIVE_FEATURES if hasattr(self.config, "DERIVE_FEATURES") else False,})
 
     def train_model(self) -> None:
         """
@@ -258,14 +254,6 @@ def start_training(self) -> None:
             save_weights_only=False,
         )  # save best model
 
-        early_stopping = callbacks.EarlyStopping(
-            monitor=self.config.CALLBACK_PARAMETER,
-            patience=int(0.3 * self.config.EPOCHS),
-            verbose=1,
-            mode="auto",
-            restore_best_weights=True,
-        )
-
         tensorboard = callbacks.TensorBoard(log_dir=str(self.config.MODEL_SAVE_DIR / "logs"), write_images=True)
 
         def lr_scheduler(epoch):
@@ -276,18 +264,25 @@ def lr_scheduler(epoch):
             else:
                 return self.config.MIN_LR
 
-        lr_callback = callbacks.LearningRateScheduler(lambda epoch: lr_scheduler(epoch), verbose=True)
-
         model_callbacks = [model_checkpoint, tensorboard]
 
         if self.config.USE_ADJUSTED_LR:
+            lr_callback = callbacks.LearningRateScheduler(lambda epoch: lr_scheduler(epoch), verbose=True)
             model_callbacks.append(lr_callback)
 
         if self.config.EARLY_STOPPING:
+            early_stopping = callbacks.EarlyStopping(
+                monitor=self.config.CALLBACK_PARAMETER,
+                patience=int(0.3 * self.config.EPOCHS),
+                verbose=1,
+                mode="auto",
+                restore_best_weights=True,
+            )
             model_callbacks.append(early_stopping)
 
         self.model_callbacks = model_callbacks
 
+        print(self.model)
         self.history = self.model.fit(
             x=self.TRAINING_DATASET,
             epochs=self.config.EPOCHS,