Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[serving] Fixes invalid CUDA_VISIBLE_DEVICE bug #903

Merged
merged 1 commit into from
Jul 5, 2023
+9 −6
Merged

[serving] Fixes invalid CUDA_VISIBLE_DEVICE bug #903

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions engines/python/src/main/java/ai/djl/python/engine/Connection.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,9 @@ class Connection {
private Channel channel;
private RequestHandler requestHandler;

Connection(PyEnv pyEnv, int workerId, int rank) {
Connection(PyEnv pyEnv, int basePort, int rank) {
requestHandler = new RequestHandler();
port = 19000 + workerId;
port = 19000 + basePort;
socketAddress = getSocketAddress(pyEnv.isMpiMode(), rank);
}

Expand Down Expand Up @@ -99,6 +99,8 @@ CompletableFuture<Output> send(Input input) throws InterruptedException {
static String[] getPythonStartCmd(PyEnv pyEnv, Model model, int workerId, int port) {
int tensorParallelDegree = pyEnv.getTensorParallelDegree();
if (pyEnv.isMpiMode()) {
String cudaDevices = getVisibleDevices(workerId, tensorParallelDegree);
logger.info("Set CUDA_VISIBLE_DEVICES={}", cudaDevices);
String[] args = new String[36];
args[0] = "mpirun";
args[1] = "-N";
Expand All @@ -122,7 +124,7 @@ static String[] getPythonStartCmd(PyEnv pyEnv, Model model, int workerId, int po
args[16] = "-x";
args[17] = "PYTHONPATH";
args[18] = "-x";
args[19] = "CUDA_VISIBLE_DEVICES=" + getVisibleDevices(workerId, tensorParallelDegree);
args[19] = "CUDA_VISIBLE_DEVICES=" + cudaDevices;
args[20] = "-x";
args[21] = "MASTER_ADDR=" + MASTER_ADDR;
args[22] = "-x";
Expand Down
7 changes: 4 additions & 3 deletions engines/python/src/main/java/ai/djl/python/engine/PyProcess.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,16 @@ class PyProcess {
PyProcess(Model model, PyEnv pyEnv, int workerId) {
this.model = model;
this.pyEnv = pyEnv;
this.workerId = workerId + counter.getAndIncrement();
this.workerId = workerId;
int port = workerId + counter.getAndIncrement();
if (pyEnv.isMpiMode()) {
int tensorParallelDegree = pyEnv.getTensorParallelDegree();
connections = new ArrayList<>(tensorParallelDegree);
for (int i = 0; i < tensorParallelDegree; ++i) {
connections.add(new Connection(pyEnv, this.workerId, i));
connections.add(new Connection(pyEnv, port, i));
}
} else {
connections = Collections.singletonList(new Connection(pyEnv, this.workerId, -1));
connections = Collections.singletonList(new Connection(pyEnv, port, -1));
}
restartCount = new AtomicInteger(0);
}
Expand Down
Loading