feat: fix Dockerfile to vllm-openai v0.5.5

hommayushi3 · Aug 25, 2024 · fa84071 · fa84071
1 parent d052c04
commit fa84071
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM vllm/vllm-openai:latest
+FROM vllm/vllm-openai:v0.5.5
 
 ENV DO_NOT_TRACK=1
 

diff --git a/endpoints-entrypoint.sh b/endpoints-entrypoint.sh
@@ -12,6 +12,7 @@ TRUST_REMOTE_CODE=${TRUST_REMOTE_CODE:-false}
 GUIDED_DECODING_BACKEND=${GUIDED_DECODING_BACKEND:-"outlines"}
 KV_CACHE_DTYPE=${KV_CACHE_DTYPE:-"auto"}
 ENFORCE_EAGER=${ENFORCE_EAGER:-false}
+USE_V2_BLOCK_MANAGER=${USE_V2_BLOCK_MANAGER:-false}
 VLLM_ATTENTION_BACKEND=${VLLM_ATTENTION_BACKEND:-"FLASH_ATTN"}
 GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.9}
 
@@ -46,6 +47,9 @@ fi
 if [ "$ENFORCE_EAGER" = true ]; then
     CMD="$CMD --enforce-eager"
 fi
+if [ "$USE_V2_BLOCK_MANAGER" = true ]; then
+    CMD="$CMD --use-v2-block-manager"
+fi
 
 # Execute the command
 eval $CMD