From bbce13684964576819508f67bc93db024e1974c9 Mon Sep 17 00:00:00 2001 From: Alexandre Strube Date: Mon, 23 Sep 2024 19:21:17 +0200 Subject: [PATCH] Fix #3501 VLLM needs top_j to be int, not float --- fastchat/serve/vllm_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/vllm_worker.py b/fastchat/serve/vllm_worker.py index 0af680bb5f..f6a9932a53 100644 --- a/fastchat/serve/vllm_worker.py +++ b/fastchat/serve/vllm_worker.py @@ -71,7 +71,7 @@ async def generate_stream(self, params): request_id = params.pop("request_id") temperature = float(params.get("temperature", 1.0)) top_p = float(params.get("top_p", 1.0)) - top_k = params.get("top_k", -1.0) + top_k = params.get("top_k", -1) presence_penalty = float(params.get("presence_penalty", 0.0)) frequency_penalty = float(params.get("frequency_penalty", 0.0)) max_new_tokens = params.get("max_new_tokens", 256)