Sync

.devcontainer/onCreateCommand.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 python -m ensurepip # ensures pip is installed in the current environment
-pip install --upgrade pip
+pip install --upgrade pip==24.0
 pip install wheel
 pip install openvino-dev==2023.0.1 # [OPTIONAL] to generate optimized models for inference
 pip install mlcube_docker          # [OPTIONAL] to deploy GaNDLF models as MLCube-compliant Docker containers

.github/workflows/black.yml

@@ -27,7 +27,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade pip==24.0
           python -m pip install black==${{ env.BLACK_VERSION }}
 
       - name: Run tests

.github/workflows/main.yml

@@ -38,7 +38,7 @@ jobs:
           ${{ runner.os }}-pip-
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade pip==24.0
         pip install scikit-build
         pip install -e .
         pip install build

.github/workflows/mlcube-test.yml

@@ -70,7 +70,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install libvips libvips-tools -y
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade pip==24.0
         python -m pip install wheel
         python -m pip install openvino-dev==2023.0.1 mlcube_docker
         pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cpu

.github/workflows/openfl-test.yml

@@ -71,7 +71,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install libvips libvips-tools -y
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade pip==24.0
         python -m pip install wheel
         pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cpu
         pip install -e .

.github/workflows/publish-nightly.yml

@@ -52,7 +52,7 @@ jobs:
     - name: Install dependencies
       if: env.publish_nightly
       run: |
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade pip==24.0
         pip install scikit-build
         pip install -e .
         pip install build

.github/workflows/python-test.yml

@@ -71,7 +71,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install libvips libvips-tools -y
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade pip==24.0
         python -m pip install wheel
         python -m pip install openvino-dev==2023.0.1 mlcube_docker
         pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cpu

Dockerfile-CPU

@@ -7,7 +7,7 @@ LABEL version=1.0
 RUN apt-get update && apt-get install -y software-properties-common
 RUN add-apt-repository ppa:deadsnakes/ppa
 RUN apt-get update &&  apt-get install -y  python3.9 python3-pip libjpeg8-dev zlib1g-dev python3-dev libpython3.9-dev libffi-dev libgl1
-RUN python3.9 -m pip install --upgrade pip
+RUN python3.9 -m pip install --upgrade pip==24.0
 # EXPLICITLY install cpu versions of torch/torchvision (not all versions have +cpu modes on PyPI...)
 RUN python3.9 -m pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cpu
 RUN python3.9 -m pip install openvino-dev==2023.0.1 opencv-python-headless mlcube_docker

Dockerfile-CUDA11.8

@@ -11,7 +11,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y software-properties-common
 RUN add-apt-repository ppa:deadsnakes/ppa
 RUN apt-get update &&  apt-get install -y  python3.9 python3-pip libjpeg8-dev zlib1g-dev python3-dev libpython3.9-dev libffi-dev libgl1
-RUN python3.9 -m pip install --upgrade pip
+RUN python3.9 -m pip install --upgrade pip==24.0
 RUN python3.9 -m pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu118
 RUN python3.9 -m pip install openvino-dev==2023.0.1 opencv-python-headless mlcube_docker
 

Dockerfile-CUDA12.1

@@ -11,7 +11,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y software-properties-common
 RUN add-apt-repository ppa:deadsnakes/ppa
 RUN apt-get update &&  apt-get install -y  python3.9 python3-pip libjpeg8-dev zlib1g-dev python3-dev libpython3.9-dev libffi-dev libgl1
-RUN python3.9 -m pip install --upgrade pip
+RUN python3.9 -m pip install --upgrade pip==24.0
 RUN python3.9 -m pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu121
 RUN python3.9 -m pip install openvino-dev==2023.0.1 opencv-python-headless mlcube_docker
 

Dockerfile-ROCm

@@ -9,9 +9,9 @@ LABEL version=1.0
 RUN apt-get update && apt-get install -y software-properties-common
 RUN add-apt-repository ppa:deadsnakes/ppa
 RUN apt-get update &&  apt-get install -y  python3.9 python3-pip libjpeg8-dev zlib1g-dev python3-dev libpython3.9-dev libffi-dev libgl1
-RUN python3.9 -m pip install --upgrade pip
+RUN python3.9 -m pip install --upgrade pip==24.0
 RUN python3.9 -m pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/rocm5.6
-RUN python3.9 -m pip install --upgrade pip && python3.9 -m pip install openvino-dev==2023.0.1 opencv-python-headless mlcube_docker
+RUN python3.9 -m pip install --upgrade pip==24.0 && python3.9 -m pip install openvino-dev==2023.0.1 opencv-python-headless mlcube_docker
 RUN apt-get update && apt-get install -y libgl1
 
 # Do some dependency installation separately here to make layer caching more efficient

GANDLF/compute/forward_pass.py

@@ -1,6 +1,6 @@
 import os
 import pathlib
-from typing import Optional, Tuple
+from typing import Optional, Tuple, Union
 
 import numpy as np
 import pandas as pd
@@ -51,21 +51,20 @@ def validate_network(
     print("*" * 20)
     # Initialize a few things
     total_epoch_valid_loss = 0
-    total_epoch_valid_metric = {}
+    total_epoch_valid_metric: dict[str, Union[float, np.array]] = {}
     average_epoch_valid_metric = {}
 
     for metric in params["metrics"]:
         if "per_label" in metric:
-            total_epoch_valid_metric[metric] = []
+            total_epoch_valid_metric[metric] = np.zeros(1)
         else:
             total_epoch_valid_metric[metric] = 0
 
     logits_list = []
     subject_id_list = []
     is_classification = params.get("problem_type") == "classification"
     calculate_overall_metrics = (
-        (params["problem_type"] == "classification")
-        or (params["problem_type"] == "regression")
+        params["problem_type"] in {"classification", "regression"}
     ) and mode == "validation"
     is_inference = mode == "inference"
 
@@ -107,10 +106,8 @@ def validate_network(
 
     # get ground truths for classification problem, validation set
     if calculate_overall_metrics:
-        (
-            ground_truth_array,
-            predictions_array,
-        ) = get_ground_truths_and_predictions_tensor(params, "validation_data")
+        ground_truth_array = []
+        predictions_array = []
 
     for batch_idx, (subject) in enumerate(
         tqdm(valid_dataloader, desc="Looping over " + mode + " data")
@@ -193,6 +190,7 @@ def validate_network(
 
             if params["save_output"] or is_inference:
                 # we divide by scaling factor here because we multiply by it during loss/metric calculation
+                # TODO: regression-only, right?
                 outputToWrite += (
                     str(epoch)
                     + ","
@@ -206,23 +204,15 @@ def validate_network(
             )
 
             if calculate_overall_metrics:
-                predictions_array[batch_idx] = (
-                    torch.argmax(pred_output[0], 0).cpu().item()
-                )
+                ground_truth_array.append(label_ground_truth.item())
+                # TODO: that's for classification only. What about regression?
+                predictions_array.append(torch.argmax(pred_output[0], 0).cpu().item())
             # # Non network validation related
             total_epoch_valid_loss += final_loss.detach().cpu().item()
-            for metric in final_metric.keys():
-                if isinstance(total_epoch_valid_metric[metric], list):
-                    if len(total_epoch_valid_metric[metric]) == 0:
-                        total_epoch_valid_metric[metric] = np.array(
-                            final_metric[metric]
-                        )
-                    else:
-                        total_epoch_valid_metric[metric] += np.array(
-                            final_metric[metric]
-                        )
-                else:
-                    total_epoch_valid_metric[metric] += final_metric[metric]
+            for metric, metric_val in final_metric.items():
+                total_epoch_valid_metric[metric] = (
+                    total_epoch_valid_metric[metric] + metric_val
+                )
 
         else:  # for segmentation problems OR regression/classification when no label is present
             grid_sampler = torchio.inference.GridSampler(
@@ -315,8 +305,7 @@ def validate_network(
 
             # save outputs
             if params["problem_type"] == "segmentation":
-                output_prediction = aggregator.get_output_tensor()
-                output_prediction = output_prediction.unsqueeze(0)
+                output_prediction = aggregator.get_output_tensor().unsqueeze(0)
                 if params["save_output"]:
                     img_for_metadata = torchio.ScalarImage(
                         tensor=subject["1"]["data"].squeeze(0),
@@ -386,16 +375,18 @@ def validate_network(
                 # final regression output
                 output_prediction = output_prediction / len(patch_loader)
                 if calculate_overall_metrics:
-                    predictions_array[batch_idx] = (
+                    # TOD: what? regression and argmax?
+                    predictions_array.append(
                         torch.argmax(output_prediction[0], 0).cpu().item()
                     )
+                    ground_truth_array.append(label_ground_truth.item())
                 if params["save_output"]:
                     outputToWrite += (
                         str(epoch)
                         + ","
                         + subject["subject_id"][0]
                         + ","
-                        + str(output_prediction)
+                        + str(output_prediction[0])
                         + "\n"
                     )
 
@@ -407,7 +398,6 @@ def validate_network(
                         n.squeeze(), raw_input=image[i].squeeze(-1)
                     )
 
-            output_prediction = output_prediction.squeeze(-1)
             if is_inference and is_classification:
                 logits_list.append(output_prediction)
                 subject_id_list.append(subject.get("subject_id")[0])
@@ -418,9 +408,8 @@ def validate_network(
                 if label_ground_truth.shape[0] == 3:
                     label_ground_truth = label_ground_truth[0, ...].unsqueeze(0)
                 # we always want the ground truth to be in the same format as the prediction
+                # add batch dim
                 label_ground_truth = label_ground_truth.unsqueeze(0)
-                if label_ground_truth.shape[-1] == 1:
-                    label_ground_truth = label_ground_truth.squeeze(-1)
                 final_loss, final_metric = get_loss_and_metrics(
                     image,
                     label_ground_truth,
@@ -440,17 +429,9 @@ def validate_network(
                 # loss.cpu().data.item()
                 total_epoch_valid_loss += final_loss.cpu().item()
                 for metric in final_metric.keys():
-                    if isinstance(total_epoch_valid_metric[metric], list):
-                        if len(total_epoch_valid_metric[metric]) == 0:
-                            total_epoch_valid_metric[metric] = np.array(
-                                final_metric[metric]
-                            )
-                        else:
-                            total_epoch_valid_metric[metric] += np.array(
-                                final_metric[metric]
-                            )
-                    else:
-                        total_epoch_valid_metric[metric] += final_metric[metric]
+                    total_epoch_valid_metric[metric] = (
+                        total_epoch_valid_metric[metric] + final_metric[metric]
+                    )
 
         if label_ground_truth is not None:
             if params["verbose"]:
@@ -486,7 +467,9 @@ def validate_network(
         # get overall stats for classification
         if calculate_overall_metrics:
             average_epoch_valid_metric = overall_stats(
-                predictions_array, ground_truth_array, params
+                torch.Tensor(predictions_array),
+                torch.Tensor(ground_truth_array),
+                params,
             )
         average_epoch_valid_metric = print_and_format_metrics(
             average_epoch_valid_metric,

GANDLF/compute/loss_and_metric.py

@@ -1,5 +1,6 @@
 import sys
-from typing import Dict, Tuple
+import warnings
+from typing import Dict, Tuple, Union
 from GANDLF.losses import global_losses_dict
 from GANDLF.metrics import global_metrics_dict
 import torch
@@ -13,7 +14,7 @@ def get_metric_output(
     prediction: torch.Tensor,
     target: torch.Tensor,
     params: dict,
-) -> float:
+) -> Union[float, list]:
     """
     This function computes the metric output for a given metric function, prediction and target.
 
@@ -36,6 +37,12 @@ def get_metric_output(
         if len(temp) > 1:
             return temp
         else:
+            # TODO: this branch is extremely age case and is buggy.
+            #  Overall the case when metric returns a list but of length 1 is very rare. The only case is when
+            #  the metric returns Nx.. tensor (i.e. without aggregation by elements) and batch_size==N==1. This branch
+            #  would definitely fail for such a metrics like
+            #  MulticlassAccuracy(num_classes=3, multidim_average="samplewise")
+            #  Maybe the best solution is to raise an error here if metric is configured to return samplewise results?
             return metric_output.item()
 
 
@@ -115,41 +122,38 @@ def get_loss_and_metrics(
         loss_kld = global_losses_dict["kld"](prediction[2], prediction[3])
         loss_cycle = global_losses_dict["mse"](prediction[2], prediction[4], None)
         loss = 0.01 * loss_kld + loss_reco + 10 * loss_seg + loss_cycle
+    elif deep_supervision_model:
+        # this is for models that have deep-supervision
+        for i, _ in enumerate(prediction):
+            # loss is calculated based on resampled "soft" labels using a pre-defined weights array
+            loss += (
+                loss_function(prediction[i], ground_truth_resampled[i], params)
+                * loss_weights[i]
+            )
     else:
-        if deep_supervision_model:
-            # this is for models that have deep-supervision
-            for i, _ in enumerate(prediction):
-                # loss is calculated based on resampled "soft" labels using a pre-defined weights array
-                loss += (
-                    loss_function(prediction[i], ground_truth_resampled[i], params)
-                    * loss_weights[i]
-                )
-        else:
-            loss = loss_function(prediction, target, params)
+        loss = loss_function(prediction, target, params)
     metric_output = {}
 
     # Metrics should be a list
     for metric in params["metrics"]:
         metric_lower = metric.lower()
         metric_output[metric] = 0
-        if metric_lower in global_metrics_dict:
-            metric_function = global_metrics_dict[metric_lower]
-            if sdnet_check:
-                metric_output[metric] = get_metric_output(
-                    metric_function, prediction[0], target.squeeze(-1), params
+        if metric_lower not in global_metrics_dict:
+            warnings.warn("WARNING: Could not find the requested metric '" + metric)
+            continue
+
+        metric_function = global_metrics_dict[metric_lower]
+        if sdnet_check:
+            metric_output[metric] = get_metric_output(
+                metric_function, prediction[0], target.squeeze(-1), params
+            )
+        elif deep_supervision_model:
+            for i, _ in enumerate(prediction):
+                metric_output[metric] += get_metric_output(
+                    metric_function, prediction[i], ground_truth_resampled[i], params
                 )
-            else:
-                if deep_supervision_model:
-                    for i, _ in enumerate(prediction):
-                        metric_output[metric] += get_metric_output(
-                            metric_function,
-                            prediction[i],
-                            ground_truth_resampled[i],
-                            params,
-                        )
-
-                else:
-                    metric_output[metric] = get_metric_output(
-                        metric_function, prediction, target, params
-                    )
+        else:
+            metric_output[metric] = get_metric_output(
+                metric_function, prediction, target, params
+            )
     return loss, metric_output