update leaderboard and documentation with WSI segmentation datasets

README.md

@@ -214,27 +214,28 @@ and [tutorials](https://kaiko-ai.github.io/eva/dev/user-guide/advanced/replicate
 
 In this section you will find model benchmarks which were generated with _`eva`_.
 
-### Table I: WSI patch-level classification tasks
+### Table I: WSI and microscopy image tasks
 
 <br />
 
 <div align="center">
 
-| Model   | BACH  | CRC   | MHIST | PCam   | Camelyon16 | PANDA |
-|---------|-------|-------|-------|--------|------------|-------|
-| ViT-S/16 _(random)_	<sup>[1]</sup> | 0.411|0.613|0.5|0.752|0.551|0.347|
-| ViT-S/16 _(ImageNet)_ <sup>[1]</sup> | 0.675|0.936|0.827|0.861|0.751|0.676|
-| DINO<sub>(p=16)</sub> <sup>[2]</sup> | 0.77|0.936|0.751|0.905|0.869|0.737|
-| Phikon <sup>[3]</sup>                | 0.715|0.942|0.766|0.925|0.879|0.784|
-| UNI <sup>[4]</sup>                   | 0.797|0.95|0.835|0.939|0.933|0.774|
-| ViT-S/16 _(kaiko.ai)_ <sup>[5]</sup> | 0.8|0.949|0.831|0.902|0.897|0.77|
-| ViT-S/8 _(kaiko.ai)_ <sup>[5]</sup>  | 0.825|0.948|0.826|0.887|0.879|0.741|
-| ViT-B/16 _(kaiko.ai)_	<sup>[5]</sup> | 0.846|0.959|0.839|0.906|0.891|0.753|
-| ViT-B/8 _(kaiko.ai)_ <sup>[5]</sup>  | 0.867|0.952|0.814|0.921|0.939|0.761|
-| ViT-L/14 _(kaiko.ai)_ <sup>[5]</sup> | 0.862|0.935|0.822|0.907|0.941|0.769|
-
-_Table I: Linear probing evaluation of FMs on patch-level downstream datasets.<br> We report averaged balanced accuracy
-over 5 runs. Results are reported on the "test" split if available and otherwise on the "validation" split._
+| Model   | BACH  | CRC   | MHIST | PCam   | Camelyon16 | PANDA | CoNSeP | MoNuSAC |
+|---------|-------|-------|-------|--------|------------|-------|------------|-------|
+| ViT-S/16 _(random)_	<sup>[1]</sup> | 0.411|0.613|0.5|0.752|0.551|0.347|0.489|0.394|
+| ViT-S/16 _(ImageNet)_ <sup>[1]</sup> | 0.675|0.936|0.827|0.861|0.751|0.676|0.54|0.512|
+| DINO<sub>(p=16)</sub> <sup>[2]</sup> | 0.77|0.936|0.751|0.905|0.869|0.737|0.625|0.549|
+| Phikon <sup>[3]</sup>                | 0.715|0.942|0.766|0.925|0.879|0.784|0.68|0.554|
+| UNI <sup>[4]</sup>                   | 0.797|0.95|0.835|0.939|0.933|0.774|0.67|0.575|
+| ViT-S/16 _(kaiko.ai)_ <sup>[5]</sup> | 0.8|0.949|0.831|0.902|0.897|0.77|0.622|0.573|
+| ViT-S/8 _(kaiko.ai)_ <sup>[5]</sup>  | 0.825|0.948|0.826|0.887|0.879|0.741|0.677|0.617|
+| ViT-B/16 _(kaiko.ai)_	<sup>[5]</sup> | 0.846|0.959|0.839|0.906|0.891|0.753|0.647|0.572|
+| ViT-B/8 _(kaiko.ai)_ <sup>[5]</sup>  | 0.867|0.952|0.814|0.921|0.939|0.761|0.706|0.661|
+| ViT-L/14 _(kaiko.ai)_ <sup>[5]</sup> | 0.862|0.935|0.822|0.907|0.941|0.769|0.686|0.599|
+
+_Table I: Linear probing evaluation of FMs on patch-level downstream datasets.<br> We report balanced accuracy
+ for classification tasks and generalized DICE score for semgetnation tasks, averaged over 5 runs. Results are 
+ reported on the "test" split if available and otherwise on the "validation" split._
 
 </div>
 

configs/vision/dino_vit/offline/bcss.yaml

@@ -4,7 +4,7 @@ trainer:
   init_args:
     n_runs: &N_RUNS ${oc.env:N_RUNS, 1}
     default_root_dir: &OUTPUT_ROOT ${oc.env:OUTPUT_ROOT, logs/${oc.env:TIMM_MODEL_NAME, vit_small_patch16_224}/bcss}
-    max_steps: &MAX_STEPS ${oc.env:MAX_STEPS, 513}
+    max_steps: &MAX_STEPS ${oc.env:MAX_STEPS, 2000}
     log_every_n_steps: 6
     callbacks:
       - class_path: eva.callbacks.ConfigurationLogger
@@ -22,7 +22,7 @@ trainer:
       - class_path: lightning.pytorch.callbacks.EarlyStopping
         init_args:
           min_delta: 0
-          patience: 100
+          patience: 8
           monitor: *MONITOR_METRIC
           mode: *MONITOR_METRIC_MODE
       - class_path: eva.callbacks.SegmentationEmbeddingsWriter
@@ -31,7 +31,9 @@ trainer:
           dataloader_idx_map:
             0: train
             1: val
+            2: test
           metadata_keys: ["coords"]
+          overwrite: true
           backbone:
             class_path: eva.vision.models.networks.encoders.TimmEncoder
             init_args:
@@ -62,7 +64,7 @@ model:
     optimizer:
       class_path: torch.optim.AdamW
       init_args:
-        lr: 0.002
+        lr: 0.0001
     lr_scheduler:
       class_path: torch.optim.lr_scheduler.PolynomialLR
       init_args:
@@ -104,6 +106,11 @@ data:
         init_args:
           <<: *DATASET_ARGS
           split: val
+      test:
+        class_path: eva.vision.datasets.EmbeddingsSegmentationDataset
+        init_args:
+          <<: *DATASET_ARGS
+          split: test
       predict:
         - class_path: eva.vision.datasets.BCSS
           init_args: &PREDICT_DATASET_ARGS

configs/vision/dino_vit/offline/consep.yaml

@@ -4,7 +4,7 @@ trainer:
   init_args:
     n_runs: &N_RUNS ${oc.env:N_RUNS, 1}
     default_root_dir: &OUTPUT_ROOT ${oc.env:OUTPUT_ROOT, logs/${oc.env:TIMM_MODEL_NAME, vit_small_patch16_224_dino}/consep}
-    max_steps: &MAX_STEPS ${oc.env:MAX_STEPS, 513}
+    max_steps: &MAX_STEPS ${oc.env:MAX_STEPS, 2000}
     log_every_n_steps: 6
     callbacks:
       - class_path: eva.callbacks.ConfigurationLogger
@@ -22,7 +22,7 @@ trainer:
       - class_path: lightning.pytorch.callbacks.EarlyStopping
         init_args:
           min_delta: 0
-          patience: 100
+          patience: 34
           monitor: *MONITOR_METRIC
           mode: *MONITOR_METRIC_MODE
       - class_path: eva.callbacks.SegmentationEmbeddingsWriter
@@ -32,6 +32,7 @@ trainer:
             0: train
             1: val
           metadata_keys: ["coords"]
+          overwrite: true
           backbone:
             class_path: eva.vision.models.networks.encoders.TimmEncoder
             init_args:
@@ -63,7 +64,7 @@ model:
     optimizer:
       class_path: torch.optim.AdamW
       init_args:
-        lr: 0.002
+        lr: 0.0001
     lr_scheduler:
       class_path: torch.optim.lr_scheduler.PolynomialLR
       init_args:

configs/vision/dino_vit/offline/monusac.yaml

@@ -4,10 +4,13 @@ trainer:
   init_args:
     n_runs: &N_RUNS ${oc.env:N_RUNS, 1}
     default_root_dir: &OUTPUT_ROOT ${oc.env:OUTPUT_ROOT, logs/${oc.env:TIMM_MODEL_NAME, vit_small_patch16_224_dino}/monusac}
-    max_steps: &MAX_STEPS ${oc.env:MAX_STEPS, 513}
+    max_steps: &MAX_STEPS ${oc.env:MAX_STEPS, 2000}
     log_every_n_steps: 6
     callbacks:
       - class_path: eva.callbacks.ConfigurationLogger
+      - class_path: lightning.pytorch.callbacks.LearningRateMonitor
+        init_args:
+          logging_interval: epoch
       - class_path: eva.vision.callbacks.SemanticSegmentationLogger
         init_args:
           log_every_n_epochs: 1
@@ -22,7 +25,7 @@ trainer:
       - class_path: lightning.pytorch.callbacks.EarlyStopping
         init_args:
           min_delta: 0
-          patience: 100
+          patience: 50
           monitor: *MONITOR_METRIC
           mode: *MONITOR_METRIC_MODE
       - class_path: eva.callbacks.SegmentationEmbeddingsWriter
@@ -31,6 +34,7 @@ trainer:
           dataloader_idx_map:
             0: train
             1: test
+          overwrite: true
           backbone:
             class_path: eva.vision.models.networks.encoders.TimmEncoder
             init_args:
@@ -62,7 +66,7 @@ model:
     optimizer:
       class_path: torch.optim.AdamW
       init_args:
-        lr: 0.002
+        lr: 0.0001
     lr_scheduler:
       class_path: torch.optim.lr_scheduler.PolynomialLR
       init_args:

configs/vision/phikon/offline/bcss.yaml

@@ -0,0 +1,143 @@
+---
+trainer:
+  class_path: eva.Trainer
+  init_args:
+    n_runs: &N_RUNS ${oc.env:N_RUNS, 1}
+    default_root_dir: &OUTPUT_ROOT ${oc.env:OUTPUT_ROOT, logs/phikon/bcss}
+    max_steps: &MAX_STEPS ${oc.env:MAX_STEPS, 2000}
+    log_every_n_steps: 6
+    callbacks:
+      - class_path: eva.callbacks.ConfigurationLogger
+      - class_path: eva.vision.callbacks.SemanticSegmentationLogger
+        init_args:
+          log_every_n_epochs: 1
+          log_images: false
+      - class_path: lightning.pytorch.callbacks.ModelCheckpoint
+        init_args:
+          filename: best
+          save_last: true
+          save_top_k: 1
+          monitor: &MONITOR_METRIC ${oc.env:MONITOR_METRIC, val/GeneralizedDiceScore}
+          mode: &MONITOR_METRIC_MODE ${oc.env:MONITOR_METRIC_MODE, max}
+      - class_path: lightning.pytorch.callbacks.EarlyStopping
+        init_args:
+          min_delta: 0
+          patience: 8
+          monitor: *MONITOR_METRIC
+          mode: *MONITOR_METRIC_MODE
+      - class_path: eva.callbacks.SegmentationEmbeddingsWriter
+        init_args:
+          output_dir: &DATASET_EMBEDDINGS_ROOT ${oc.env:EMBEDDINGS_ROOT, ./data/embeddings}/phikon/bcss
+          dataloader_idx_map:
+            0: train
+            1: val
+            2: test
+          metadata_keys: ["coords"]
+          overwrite: true
+          backbone:
+            class_path: eva.models.HuggingFaceModel
+            init_args:
+              model_name_or_path: owkin/phikon
+              tensor_transforms: 
+                class_path: eva.core.models.networks.transforms.ExtractPatchFeatures
+    logger:
+      - class_path: lightning.pytorch.loggers.TensorBoardLogger
+        init_args:
+          save_dir: *OUTPUT_ROOT
+          name: ""
+model:
+  class_path: eva.vision.models.modules.SemanticSegmentationModule
+  init_args:
+    decoder:
+      class_path: eva.vision.models.networks.decoders.segmentation.ConvDecoderMS
+      init_args:
+        in_features: ${oc.env:DECODER_IN_FEATURES, 768}
+        num_classes:  &NUM_CLASSES 6
+    criterion:
+      class_path: eva.vision.losses.DiceLoss
+      init_args:
+        softmax: true
+        batch: true
+    optimizer:
+      class_path: torch.optim.AdamW
+      init_args:
+        lr: 0.0001
+    lr_scheduler:
+      class_path: torch.optim.lr_scheduler.PolynomialLR
+      init_args:
+        total_iters: *MAX_STEPS
+        power: 0.9
+    postprocess:
+      predictions_transforms:
+        - class_path: torch.argmax
+          init_args:
+            dim: 1
+    metrics:
+      common:
+        - class_path: eva.metrics.AverageLoss
+      evaluation:
+        - class_path: eva.core.metrics.defaults.MulticlassSegmentationMetrics
+          init_args:
+            num_classes: *NUM_CLASSES
+        - class_path: torchmetrics.ClasswiseWrapper
+          init_args:
+            metric:
+              class_path: eva.core.metrics.GeneralizedDiceScore
+              init_args:
+                num_classes: *NUM_CLASSES
+                input_format: index
+                weight_type: linear
+                per_class: true
+data:
+  class_path: eva.DataModule
+  init_args:
+    datasets:
+      train:
+        class_path: eva.vision.datasets.EmbeddingsSegmentationDataset
+        init_args: &DATASET_ARGS
+          root: *DATASET_EMBEDDINGS_ROOT
+          manifest_file: manifest.csv
+          split: train
+      val:
+        class_path: eva.vision.datasets.EmbeddingsSegmentationDataset
+        init_args:
+          <<: *DATASET_ARGS
+          split: val
+      test:
+        class_path: eva.vision.datasets.EmbeddingsSegmentationDataset
+        init_args:
+          <<: *DATASET_ARGS
+          split: test
+      predict:
+        - class_path: eva.vision.datasets.BCSS
+          init_args: &PREDICT_DATASET_ARGS
+            root: ${oc.env:DATA_ROOT, ./data}/bcss
+            split: train
+            sampler:
+              class_path: eva.vision.data.wsi.patching.samplers.GridSampler
+              init_args:
+                max_samples: 1000
+            transforms:
+              class_path: eva.vision.data.transforms.common.ResizeAndCrop
+              init_args:
+                size: ${oc.env:RESIZE_DIM, 224}  
+                mean: &NORMALIZE_MEAN ${oc.env:NORMALIZE_MEAN, [0.485, 0.456, 0.406]} 
+                std: &NORMALIZE_STD ${oc.env:NORMALIZE_STD, [0.229, 0.224, 0.225]}
+        - class_path: eva.vision.datasets.BCSS
+          init_args:
+            <<: *PREDICT_DATASET_ARGS
+            split: val
+        - class_path: eva.vision.datasets.BCSS
+          init_args:
+            <<: *PREDICT_DATASET_ARGS
+            split: test
+    dataloaders:
+      train:
+        batch_size: &BATCH_SIZE ${oc.env:BATCH_SIZE, 64}
+        shuffle: true
+      val:
+        batch_size: *BATCH_SIZE
+      test:
+        batch_size: *BATCH_SIZE
+      predict:
+        batch_size: &PREDICT_BATCH_SIZE ${oc.env:PREDICT_BATCH_SIZE, 64}

configs/vision/phikon/offline/consep.yaml

@@ -4,7 +4,7 @@ trainer:
   init_args:
     n_runs: &N_RUNS ${oc.env:N_RUNS, 1}
     default_root_dir: &OUTPUT_ROOT ${oc.env:OUTPUT_ROOT, logs/phikon/consep}
-    max_steps: &MAX_STEPS ${oc.env:MAX_STEPS, 513}
+    max_steps: &MAX_STEPS ${oc.env:MAX_STEPS, 2000}
     log_every_n_steps: 6
     callbacks:
       - class_path: eva.callbacks.ConfigurationLogger
@@ -22,7 +22,7 @@ trainer:
       - class_path: lightning.pytorch.callbacks.EarlyStopping
         init_args:
           min_delta: 0
-          patience: 100
+          patience: 34
           monitor: *MONITOR_METRIC
           mode: *MONITOR_METRIC_MODE
       - class_path: eva.callbacks.SegmentationEmbeddingsWriter
@@ -59,7 +59,7 @@ model:
     optimizer:
       class_path: torch.optim.AdamW
       init_args:
-        lr: 0.002
+        lr: 0.0001
     lr_scheduler:
       class_path: torch.optim.lr_scheduler.PolynomialLR
       init_args: