chore: use rps for rate limit (#3493)

* chore: use rps for rate limit Signed-off-by: Wei Zhang <[email protected]> * chore: use rpm for max and init in rate limit Signed-off-by: Wei Zhang <[email protected]> --------- Signed-off-by: Wei Zhang <[email protected]>
TabbyML · Nov 29, 2024 · 6236c30 · 6236c30
1 parent 0f8a5a0
commit 6236c30
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/crates/http-api-bindings/src/rate_limit.rs b/crates/http-api-bindings/src/rate_limit.rs
@@ -12,7 +12,7 @@ use ratelimit::Ratelimiter;
 use tabby_inference::{ChatCompletionStream, CompletionOptions, CompletionStream, Embedding};
 
 fn new_rate_limiter(rpm: u64) -> Ratelimiter {
-    Ratelimiter::builder(rpm, Duration::from_secs(60))
+    Ratelimiter::builder(rpm/60, Duration::from_secs(1))
         .max_tokens(rpm)
         .initial_available(rpm)
         .build()