diff --git a/larq_zoo/training/knowledge_distillation/multi_stage_training.py b/larq_zoo/training/knowledge_distillation/multi_stage_training.py
index e751a226..e5dfa03e 100644
--- a/larq_zoo/training/knowledge_distillation/multi_stage_training.py
+++ b/larq_zoo/training/knowledge_distillation/multi_stage_training.py
@@ -179,5 +179,7 @@ def run(self) -> None:
         Path(self.parent_output_dir).mkdir(parents=True, exist_ok=True)
 
         for experiment in self.experiments:
+            if experiment.stage < self.initial_stage:
+                continue
             print(f"Starting stage {experiment.stage} at {datetime.now().isoformat()}.")
             experiment.run()