more logging (#1309)

dlrover/python/elastic_agent/torch/training.py

@@ -194,6 +194,11 @@ def auto_configure_params(self):
             device = torch.cuda.get_device_name()
         if "Ascend" in device:
             self.accelerator = Accelerators.ASCEND_NPU
+        logger.info(
+            f"Use {self.accelerator} device for training, "
+            f"cuda is available: {torch.cuda.is_available()}."
+        )
+
         if not self.auto_config:
             return
 

dlrover/trainer/torch/node_check/ascend_npu.py

@@ -42,6 +42,11 @@ def main():
         device = torch.cuda.get_device_name()
         if "Ascend" in device:
             protocol = "hccl"
+    else:
+        logger.warning(
+            f"Use GLOO as comm protocol for use_cuda: {use_cuda} "
+            "or 'Ascend' not in `torch.cuda.get_device_name()`."
+        )
 
     init_process_group(protocol, timeout=get_network_check_timeout())
 

dlrover/trainer/torch/node_check/nvidia_gpu.py

@@ -16,6 +16,8 @@
 import torch
 import torch.distributed as dist
 
+from dlrover.python.common.log import default_logger as logger
+
 from .utils import (
     bm_allreduce,
     get_network_check_timeout,
@@ -37,7 +39,12 @@ def set_nccl_env():
 @record_execution_time
 def main():
     use_cuda = torch.cuda.is_available()
-    protocol = "nccl" if use_cuda else "gloo"
+
+    if use_cuda:
+        protocol = "nccl"
+    else:
+        logger.warning("Use GLOO as comm protocol for cuda is not available.")
+        protocol = "gloo"
 
     init_process_group(protocol, timeout=get_network_check_timeout())
     if use_cuda: