diff --git a/docs/modules/usage/llms/llms.md b/docs/modules/usage/llms/llms.md index 5e6a472d0c0a..f4fa118dd02e 100644 --- a/docs/modules/usage/llms/llms.md +++ b/docs/modules/usage/llms/llms.md @@ -63,22 +63,22 @@ We have a few guides for running OpenHands with specific model providers: ### API retries and rate limits LLM providers typically have rate limits, sometimes very low, and may require retries. OpenHands will automatically -retry requests if it receives a Rate Limit Error (429 error code), API connection error, or other transient errors. +retry requests if it receives a Rate Limit Error (429 error code). You can customize these options as you need for the provider you're using. Check their documentation, and set the following environment variables to control the number of retries and the time between retries: -- `LLM_NUM_RETRIES` (Default of 8) -- `LLM_RETRY_MIN_WAIT` (Default of 15 seconds) -- `LLM_RETRY_MAX_WAIT` (Default of 120 seconds) +- `LLM_NUM_RETRIES` (Default of 4 times) +- `LLM_RETRY_MIN_WAIT` (Default of 5 seconds) +- `LLM_RETRY_MAX_WAIT` (Default of 30 seconds) - `LLM_RETRY_MULTIPLIER` (Default of 2) If you are running OpenHands in development mode, you can also set these options in the `config.toml` file: ```toml [llm] -num_retries = 8 -retry_min_wait = 15 -retry_max_wait = 120 +num_retries = 4 +retry_min_wait = 5 +retry_max_wait = 30 retry_multiplier = 2 ``` diff --git a/frontend/src/i18n/translation.json b/frontend/src/i18n/translation.json index eaa0ccf43b8e..2cec18014f58 100644 --- a/frontend/src/i18n/translation.json +++ b/frontend/src/i18n/translation.json @@ -3803,6 +3803,37 @@ "pt": "Erro ao autenticar com o provedor LLM. Por favor, verifique sua chave API", "tr": "LLM sağlayıcısı ile kimlik doğrulama hatası. Lütfen API anahtarınızı kontrol edin" }, + "STATUS$ERROR_LLM_SERVICE_UNAVAILABLE": { + "en": "The LLM provider is currently unavailable. Please try again later.", + "es": "El proveedor LLM no está actualmente disponible. Por favor, inténtelo de nuevo más tarde.", + "zh-CN": "LLM提供商当前不可用", + "zh-TW": "LLM提供商目前無法使用", + "ko-KR": "LLM 공급자가 현재 사용 불가능합니다", + "ja": "LLMプロバイダーが現在利用できません。後でもう一度試してください。", + "no": "LLM-leverandøren er nå ikke tilgjengelig. Vennligst prøv igjen senere.", + "ar": "المزود LLM غير متاح حالياً. يرجى المحاولة مرة أخرى لاحقًا.", + "de": "Der LLM-Anbieter ist derzeit nicht verfügbar. Bitte versuchen Sie es später erneut.", + "fr": "Le fournisseur LLM n'est actuellement pas disponible. Veuillez réessayer plus tard.", + "it": "Il provider LLM non è attualmente disponibile. Per favore, riprova più tardi.", + "pt": "O provedor LLM não está atualmente disponível. Por favor, tente novamente mais tarde.", + "tr": "LLM sağlayıcısı şu anda kullanılamıyor. Lütfen daha sonra tekrar deneyin." + }, + "STATUS$ERROR_LLM_INTERNAL_SERVER_ERROR": { + "en": "The request failed with an internal server error.", + "es": "La solicitud falló con un error del servidor interno.", + "zh-CN": "请求失败，请稍后再试", + "zh-TW": "請求失敗，請稍後再試", + "ko-KR": "요청이 실패했습니다. 나중에 다시 시도해주세요.", + "ja": "リクエストが内部サーバーエラーで失敗しました。後でもう一度試してください。", + "no": "Det oppstod en feil ved tilkobling til kjøretidsmiljøet. Vennligst oppdater siden.", + "ar": "حدث خطأ أثناء الاتصال بوقت التشغيل. يرجى تحديث الصفحة.", + "de": "Beim Verbinden mit der Laufzeitumgebung ist ein Fehler aufgetreten. Bitte aktualisieren Sie die Seite.", + "fr": "Une erreur s'est produite lors de la connexion à l'environnement d'exécution. Veuillez rafraîchir la page.", + "it": "Si è verificato un errore durante la connessione al runtime. Aggiorna la pagina.", + "pt": "Ocorreu um erro ao conectar ao ambiente de execução. Por favor, atualize a página.", + "tr": "Çalışma zamanına bağlanırken bir hata oluştu. Lütfen sayfayı yenileyin." + }, + "STATUS$ERROR_RUNTIME_DISCONNECTED": { "en": "There was an error while connecting to the runtime. Please refresh the page.", "zh-CN": "运行时已断开连接", @@ -3820,7 +3851,18 @@ }, "STATUS$LLM_RETRY": { "en": "Retrying LLM request", - "zh-TW": "重新嘗試 LLM 請求中" + "es": "Reintentando solicitud LLM", + "zh-CN": "重试LLM请求", + "zh-TW": "重試LLM請求", + "ko-KR": "LLM 요청 재시도", + "ja": "LLM リクエストを再試行中", + "no": "Gjenforsøker LLM-forespørsel", + "ar": "يتم إعادة تحميل الطلب LLM", + "de": "LLM-Anfrage erneut versuchen", + "fr": "Réessayer la requête LLM", + "it": "Ritenta la richiesta LLM", + "pt": "Reintentando a solicitação LLM", + "tr": "LLM isteğini yeniden deniyor" }, "AGENT_ERROR$BAD_ACTION": { "en": "Agent tried to execute a malformed action.", diff --git a/openhands/controller/agent_controller.py b/openhands/controller/agent_controller.py index e5a0b24f9694..1e338810198a 100644 --- a/openhands/controller/agent_controller.py +++ b/openhands/controller/agent_controller.py @@ -214,6 +214,17 @@ async def _react_to_exception( err_id = '' if isinstance(e, litellm.AuthenticationError): err_id = 'STATUS$ERROR_LLM_AUTHENTICATION' + elif isinstance( + e, + ( + litellm.ServiceUnavailableError, + litellm.APIConnectionError, + litellm.APIError, + ), + ): + err_id = 'STATUS$ERROR_LLM_SERVICE_UNAVAILABLE' + elif isinstance(e, litellm.InternalServerError): + err_id = 'STATUS$ERROR_LLM_INTERNAL_SERVER_ERROR' elif isinstance(e, RateLimitError): await self.set_agent_state_to(AgentState.RATE_LIMITED) return diff --git a/openhands/core/config/llm_config.py b/openhands/core/config/llm_config.py index cb1581634da1..cee22766df14 100644 --- a/openhands/core/config/llm_config.py +++ b/openhands/core/config/llm_config.py @@ -59,10 +59,11 @@ class LLMConfig(BaseModel): aws_region_name: str | None = Field(default=None) openrouter_site_url: str = Field(default='https://docs.all-hands.dev/') openrouter_app_name: str = Field(default='OpenHands') - num_retries: int = Field(default=8) + # total wait time: 5 + 10 + 20 + 30 = 65 seconds + num_retries: int = Field(default=4) retry_multiplier: float = Field(default=2) - retry_min_wait: int = Field(default=15) - retry_max_wait: int = Field(default=120) + retry_min_wait: int = Field(default=5) + retry_max_wait: int = Field(default=30) timeout: int | None = Field(default=None) max_message_chars: int = Field( default=30_000 diff --git a/openhands/llm/llm.py b/openhands/llm/llm.py index e8dd2f6f1ef8..4ff0491e322c 100644 --- a/openhands/llm/llm.py +++ b/openhands/llm/llm.py @@ -18,11 +18,7 @@ from litellm import completion as litellm_completion from litellm import completion_cost as litellm_completion_cost from litellm.exceptions import ( - APIConnectionError, - APIError, - InternalServerError, RateLimitError, - ServiceUnavailableError, ) from litellm.types.utils import CostPerToken, ModelResponse, Usage from litellm.utils import create_pretrained_tokenizer @@ -41,15 +37,7 @@ __all__ = ['LLM'] # tuple of exceptions to retry on -LLM_RETRY_EXCEPTIONS: tuple[type[Exception], ...] = ( - APIConnectionError, - # FIXME: APIError is useful on 502 from a proxy for example, - # but it also retries on other errors that are permanent - APIError, - InternalServerError, - RateLimitError, - ServiceUnavailableError, -) +LLM_RETRY_EXCEPTIONS: tuple[type[Exception], ...] = (RateLimitError,) # cache prompt supporting models # remove this when we gemini and deepseek are supported diff --git a/tests/unit/test_llm.py b/tests/unit/test_llm.py index 98783c050d0a..1bfee8550698 100644 --- a/tests/unit/test_llm.py +++ b/tests/unit/test_llm.py @@ -3,10 +3,7 @@ import pytest from litellm.exceptions import ( - APIConnectionError, - InternalServerError, RateLimitError, - ServiceUnavailableError, ) from openhands.core.config import LLMConfig @@ -187,21 +184,6 @@ def test_completion_with_mocked_logger( @pytest.mark.parametrize( 'exception_class,extra_args,expected_retries', [ - ( - APIConnectionError, - {'llm_provider': 'test_provider', 'model': 'test_model'}, - 2, - ), - ( - InternalServerError, - {'llm_provider': 'test_provider', 'model': 'test_model'}, - 2, - ), - ( - ServiceUnavailableError, - {'llm_provider': 'test_provider', 'model': 'test_model'}, - 2, - ), (RateLimitError, {'llm_provider': 'test_provider', 'model': 'test_model'}, 2), ], ) @@ -254,22 +236,6 @@ def test_completion_rate_limit_wait_time(mock_litellm_completion, default_config ), f'Expected wait time between {default_config.retry_min_wait} and {default_config.retry_max_wait} seconds, but got {wait_time}' -@patch('openhands.llm.llm.litellm_completion') -def test_completion_exhausts_retries(mock_litellm_completion, default_config): - mock_litellm_completion.side_effect = APIConnectionError( - 'Persistent error', llm_provider='test_provider', model='test_model' - ) - - llm = LLM(config=default_config) - with pytest.raises(APIConnectionError): - llm.completion( - messages=[{'role': 'user', 'content': 'Hello!'}], - stream=False, - ) - - assert mock_litellm_completion.call_count == llm.config.num_retries - - @patch('openhands.llm.llm.litellm_completion') def test_completion_operation_cancelled(mock_litellm_completion, default_config): mock_litellm_completion.side_effect = OperationCancelled('Operation cancelled') diff --git a/tests/unit/test_llm_config.py b/tests/unit/test_llm_config.py index 342112a44316..fd11deb98580 100644 --- a/tests/unit/test_llm_config.py +++ b/tests/unit/test_llm_config.py @@ -188,7 +188,7 @@ def test_load_from_toml_llm_missing_generic( assert custom_only.model == 'custom-only-model' assert custom_only.api_key.get_secret_value() == 'custom-only-api-key' assert custom_only.embedding_model == 'local' # default value - assert custom_only.num_retries == 8 # default value + assert custom_only.num_retries == 4 # default value def test_load_from_toml_llm_invalid_config(