How do I set up multi-GPU parallel training?

[noob000007]

How do I set up multi-GPU parallel training?

[noob000007]

if i use ----> 9 super(RegressionModel, mod).train(max_epochs=100, batch_size=2500, train_size=1, lr=0.002,device=4)
then will：

{
"name": "ProcessExitedException",
"message": "process 3 terminated with signal SIGSEGV",
"stack": "---------------------------------------------------------------------------
ProcessExitedException Traceback (most recent call last)
Cell In[13], line 9
1 # Use all data for training (validation not implemented yet, train_size=1)
2 # mod.train(
3 # max_epochs=100,
(...)
7 # )
8 #
----> 9 super(RegressionModel, mod).train(max_epochs=100, batch_size=2500, train_size=1, lr=0.002,device=4)

File ~/anaconda3/envs/cellana/envs/cell2loc_env/lib/python3.10/site-packages/scvi/model/base/_pyromixin.py:194, in PyroSviTrainMixin.train(self, max_epochs, accelerator, device, train_size, validation_size, shuffle_set_split, batch_size, early_stopping, lr, training_plan, datasplitter_kwargs, plan_kwargs, **trainer_kwargs)
183 trainer_kwargs["callbacks"].append(PyroJitGuideWarmup())
185 runner = self._train_runner_cls(
186 self,
187 training_plan=training_plan,
(...)
192 **trainer_kwargs,
193 )
--> 194 return runner()

File ~/anaconda3/envs/cellana/envs/cell2loc_env/lib/python3.10/site-packages/scvi/train/_trainrunner.py:96, in TrainRunner.call(self)
93 if hasattr(self.data_splitter, "n_val"):
94 self.training_plan.n_obs_validation = self.data_splitter.n_val
---> 96 self.trainer.fit(self.training_plan, self.data_splitter)
97 self._update_history()
99 # data splitter only gets these attrs after fit

File ~/anaconda3/envs/cellana/envs/cell2loc_env/lib/python3.10/site-packages/scvi/train/_trainer.py:201, in Trainer.fit(self, *args, **kwargs)
195 if isinstance(args[0], PyroTrainingPlan):
196 warnings.filterwarnings(
197 action="ignore",
198 category=UserWarning,
199 message="LightningModule.configure_optimizers returned None",
200 )
--> 201 super().fit(*args, **kwargs)

File ~/anaconda3/envs/cellana/envs/cell2loc_env/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py:538, in Trainer.fit(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)
536 self.state.status = TrainerStatus.RUNNING
537 self.training = True
--> 538 call._call_and_handle_interrupt(
539 self, self._fit_impl, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path
540 )

File ~/anaconda3/envs/cellana/envs/cell2loc_env/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py:46, in _call_and_handle_interrupt(trainer, trainer_fn, *args, **kwargs)
44 try:
45 if trainer.strategy.launcher is not None:
---> 46 return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
47 return trainer_fn(*args, **kwargs)
49 except _TunerExitException:

File ~/anaconda3/envs/cellana/envs/cell2loc_env/lib/python3.10/site-packages/lightning/pytorch/strategies/launchers/multiprocessing.py:144, in _MultiProcessingLauncher.launch(self, function, trainer, *args, **kwargs)
136 process_context = mp.start_processes(
137 self._wrapping_function,
138 args=process_args,
(...)
141 join=False, # we will join ourselves to get the process references
142 )
143 self.procs = process_context.processes
--> 144 while not process_context.join():
145 pass
147 worker_output = return_queue.get()

File ~/anaconda3/envs/cellana/envs/cell2loc_env/lib/python3.10/site-packages/torch/multiprocessing/spawn.py:170, in ProcessContext.join(self, timeout)
168 except ValueError:
169 name = f"<Unknown signal {-exitcode}>"
--> 170 raise ProcessExitedException(
171 "process %d terminated with signal %s" % (error_index, name),
172 error_index=error_index,
173 error_pid=failed_process.pid,
174 exit_code=exitcode,
175 signal_name=name,
176 )
177 else:
178 raise ProcessExitedException(
179 "process %d terminated with exit code %d" % (error_index, exitcode),
180 error_index=error_index,
181 error_pid=failed_process.pid,
182 exit_code=exitcode,
183 )

ProcessExitedException: process 3 terminated with signal SIGSEGV"
}

[vitkl]

Multi-GPU training is not needed for the regression model (batch size is never too large for most GPUs) and quite non-trivial for the cell2location model. It is non-trivial because you need to keep location-specific parameters not just location-specific data on different GPU devices and it has to be full data rather than minibatch training.