From b480a2df9588bc661fdb06e8d5cef76fc67f1acd Mon Sep 17 00:00:00 2001 From: Ran Lu Date: Thu, 16 Jan 2025 23:37:38 -0500 Subject: [PATCH] Do not autoscale deepem-gpu cluster It does not make sense to auto scale the cluster since the size is fixed --- dags/cluster_dag.py | 2 +- dags/heartbeat_dag.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dags/cluster_dag.py b/dags/cluster_dag.py index 33341830..54decbac 100644 --- a/dags/cluster_dag.py +++ b/dags/cluster_dag.py @@ -107,7 +107,7 @@ def cluster_control(): cluster_api.resize_instance_group(cluster_info[key], 0) continue - if num_workers != target_sizes[key]: + if num_workers != target_sizes[key] and key != "deepem-gpu": target_sizes[key] = max(num_workers, 1) if stable and requested_size < target_sizes[key]: diff --git a/dags/heartbeat_dag.py b/dags/heartbeat_dag.py index 6c25b769..47551c59 100644 --- a/dags/heartbeat_dag.py +++ b/dags/heartbeat_dag.py @@ -102,6 +102,8 @@ def delete_dead_instances(): r = redis.Redis(redis_host) for key in cluster_info: + if key == "deepem-gpu": + continue if target_sizes.get(key, 0) == 0: continue