From 641b134b54a718c5153206454ce99dfc26e25e00 Mon Sep 17 00:00:00 2001
From: bghira <bghira@users.github.com>
Date: Sat, 18 Jan 2025 16:03:16 -0600
Subject: [PATCH] wandb disabled command can cause error upon start

---
 helpers/training/trainer.py | 38 +++++++++++++++++++++++++------------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/helpers/training/trainer.py b/helpers/training/trainer.py
index 6b006ac3..b1b36ec4 100644
--- a/helpers/training/trainer.py
+++ b/helpers/training/trainer.py
@@ -1648,18 +1648,32 @@ def init_trackers(self):
             tracker_run_name = (
                 self.config.tracker_run_name or "simpletuner-training-run"
             )
-            self.accelerator.init_trackers(
-                project_name,
-                config=vars(public_args),
-                init_kwargs={
-                    "wandb": {
-                        "name": tracker_run_name,
-                        "id": f"{public_args_hash}",
-                        "resume": "allow",
-                        "allow_val_change": True,
-                    }
-                },
-            )
+            try:
+                self.accelerator.init_trackers(
+                    project_name,
+                    config=vars(public_args),
+                    init_kwargs={
+                        "wandb": {
+                            "name": tracker_run_name,
+                            "id": f"{public_args_hash}",
+                            "resume": "allow",
+                            "allow_val_change": True,
+                        }
+                    },
+                )
+            except Exception as e:
+                if "Object has no attribute 'disabled'" in repr(e):
+                    logger.warning(
+                        "WandB is disabled, and Accelerate was not quite happy about it."
+                    )
+                else:
+                    logger.error(f"Could not initialize trackers: {e}")
+                    self._send_webhook_raw(
+                        structured_data={
+                            "message": f"Could not initialize trackers. Continuing without. {e}"
+                        },
+                        message_type="error",
+                    )
             self._send_webhook_raw(
                 structured_data=public_args.__dict__,
                 message_type="training_config",