change timeout for modal

EmbeddedLLM · Jul 1, 2024 · 88f57af · 88f57af
1 parent 3605cad
commit 88f57af
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/infra/modal/webserver.py b/infra/modal/webserver.py
@@ -56,7 +56,7 @@ def download_models():
 # Run a web server on port 7997 and expose the Infinity embedding server
 @app.function(
     # allow up to 16 requests pending on a single container
-    allow_concurrent_inputs=16,
+    allow_concurrent_inputs=8,
     # boots take around 30 seconds, so keep containers alive for a few times longer
     container_idle_timeout=3 * MINUTES,
     # max 5 container instances