From 27b96ae4fdcd3313a6ac88a4d5b6ed4dc0b17692 Mon Sep 17 00:00:00 2001 From: Ran Lu Date: Sun, 3 Mar 2024 13:17:19 -0500 Subject: [PATCH] GCE GPU workers cannot migrate They have to be terminated on host maintaince events. --- cloud/google/workers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloud/google/workers.py b/cloud/google/workers.py index 2e45051f..10d5c340 100644 --- a/cloud/google/workers.py +++ b/cloud/google/workers.py @@ -158,6 +158,8 @@ def GenerateWorkers(context, hostname_manager, hostname_nfs_server, worker): 'acceleratorCount': accelerator_count, 'acceleratorType': worker['gpuWorkerAcceleratorType'], }] + if not worker.get('preemptible', False): + instance_template['scheduling']['onHostMaintenance'] = 'TERMINATE' if worker['type'] == "atomic": instance_template['advancedMachineFeatures'] = {