From 8ea71c2da76fd00579aeadfbe5d46360392a3d6f Mon Sep 17 00:00:00 2001
From: Engel Nyst <enyst@users.noreply.github.com>
Date: Tue, 15 Oct 2024 22:47:18 +0200
Subject: [PATCH 01/11] fix llm_config fallback

---
 openhands/core/config/utils.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/openhands/core/config/utils.py b/openhands/core/config/utils.py
index 15b64eb6d61b..1afc82dc4fd6 100644
--- a/openhands/core/config/utils.py
+++ b/openhands/core/config/utils.py
@@ -136,18 +136,30 @@ def load_from_toml(cfg: AppConfig, toml_file: str = 'config.toml'):
                     logger.openhands_logger.debug(
                         'Attempt to load default LLM config from config toml'
                     )
-                    non_dict_fields = {
+                    # Extract generic LLM fields
+                    generic_llm_fields = {
                         k: v for k, v in value.items() if not isinstance(v, dict)
                     }
-                    llm_config = LLMConfig(**non_dict_fields)
-                    cfg.set_llm_config(llm_config, 'llm')
+                    generic_llm_config = LLMConfig(**generic_llm_fields)
+                    cfg.set_llm_config(generic_llm_config, 'llm')
+
+                    # Process custom named LLM configs
                     for nested_key, nested_value in value.items():
                         if isinstance(nested_value, dict):
                             logger.openhands_logger.debug(
-                                f'Attempt to load group {nested_key} from config toml as llm config'
+                                f'Attempt to load group {nested_key} from config toml as LLM config'
                             )
-                            llm_config = LLMConfig(**nested_value)
-                            cfg.set_llm_config(llm_config, nested_key)
+                            # Apply generic LLM config with custom LLM overrides, e.g.
+                            # [llm]
+                            # model="..."
+                            # num_retries = 5
+                            # [llm.claude]
+                            # model="claude-3-5-sonnet"
+                            # results in num_retries APPLIED to claude-3-5-sonnet
+                            merged_llm_dict = generic_llm_config.__dict__.copy()
+                            merged_llm_dict.update(nested_value)
+                            custom_llm_config = LLMConfig(**merged_llm_dict)
+                            cfg.set_llm_config(custom_llm_config, nested_key)
                 elif not key.startswith('sandbox') and key.lower() != 'core':
                     logger.openhands_logger.warning(
                         f'Unknown key in {toml_file}: "{key}"'

From 17caad3b98dc77cafba791cd87fed64b57d28bb0 Mon Sep 17 00:00:00 2001
From: Engel Nyst <enyst@users.noreply.github.com>
Date: Wed, 16 Oct 2024 02:51:18 +0200
Subject: [PATCH 02/11] unit tests

---
 tests/unit/test_llm_config.py | 228 ++++++++++++++++++++++++++++++++++
 1 file changed, 228 insertions(+)
 create mode 100644 tests/unit/test_llm_config.py

diff --git a/tests/unit/test_llm_config.py b/tests/unit/test_llm_config.py
new file mode 100644
index 000000000000..2fc22d6f2232
--- /dev/null
+++ b/tests/unit/test_llm_config.py
@@ -0,0 +1,228 @@
+import pathlib
+
+import pytest
+
+from openhands.core.config import AppConfig
+from openhands.core.config.utils import load_from_toml
+
+
+@pytest.fixture
+def default_config(monkeypatch):
+    # Fixture to provide a default AppConfig instance
+    yield AppConfig()
+
+
+@pytest.fixture
+def generic_llm_toml(tmp_path: pathlib.Path) -> str:
+    """Fixture to create a generic LLM TOML configuration with all custom LLMs
+    providing mandatory 'model' and 'api_key', and testing fallback to the generic section values
+    for other attributes like 'num_retries'.
+    """
+    toml_content = """
+[core]
+workspace_base = "./workspace"
+
+[llm]
+model = "base-model"
+api_key = "base-api-key"
+embedding_model = "base-embedding"
+num_retries = 3
+
+[llm.custom1]
+model = "custom-model-1"
+api_key = "custom-api-key-1"
+# 'num_retries' is not overridden and should fallback to the value from [llm]
+
+[llm.custom2]
+model = "custom-model-2"
+api_key = "custom-api-key-2"
+num_retries = 5  # Overridden value
+
+[llm.custom3]
+model = "custom-model-3"
+api_key = "custom-api-key-3"
+# No overrides for additional attributes
+    """
+    toml_file = tmp_path / 'llm_config.toml'
+    toml_file.write_text(toml_content)
+    return str(toml_file)
+
+
+def test_load_from_toml_llm_with_fallback(
+    default_config: AppConfig, generic_llm_toml: str
+) -> None:
+    """Test that custom LLM configurations fallback non-overridden attributes
+    like 'num_retries' from the generic [llm] section.
+    """
+    load_from_toml(default_config, generic_llm_toml)
+
+    # Verify generic LLM configuration
+    generic_llm = default_config.get_llm_config('llm')
+    assert generic_llm.model == 'base-model'
+    assert generic_llm.api_key == 'base-api-key'
+    assert generic_llm.embedding_model == 'base-embedding'
+    assert generic_llm.num_retries == 3
+
+    # Verify custom1 LLM falls back 'num_retries' from base
+    custom1 = default_config.get_llm_config('custom1')
+    assert custom1.model == 'custom-model-1'
+    assert custom1.api_key == 'custom-api-key-1'
+    assert custom1.embedding_model == 'base-embedding'
+    assert custom1.num_retries == 3  # from [llm]
+
+    # Verify custom2 LLM overrides 'num_retries'
+    custom2 = default_config.get_llm_config('custom2')
+    assert custom2.model == 'custom-model-2'
+    assert custom2.api_key == 'custom-api-key-2'
+    assert custom2.embedding_model == 'base-embedding'
+    assert custom2.num_retries == 5  # overridden value
+
+    # Verify custom3 LLM inherits all attributes except 'model' and 'api_key'
+    custom3 = default_config.get_llm_config('custom3')
+    assert custom3.model == 'custom-model-3'
+    assert custom3.api_key == 'custom-api-key-3'
+    assert custom3.embedding_model == 'base-embedding'
+    assert custom3.num_retries == 3  # from [llm]
+
+
+def test_load_from_toml_llm_custom_overrides_all(
+    default_config: AppConfig, tmp_path: pathlib.Path
+) -> None:
+    """Test that a custom LLM can fully override all attributes from the generic [llm] section."""
+    toml_content = """
+[core]
+workspace_base = "./workspace"
+
+[llm]
+model = "base-model"
+api_key = "base-api-key"
+embedding_model = "base-embedding"
+num_retries = 3
+
+[llm.custom_full]
+model = "full-custom-model"
+api_key = "full-custom-api-key"
+embedding_model = "full-custom-embedding"
+num_retries = 10
+    """
+    toml_file = tmp_path / 'full_override_llm.toml'
+    toml_file.write_text(toml_content)
+
+    load_from_toml(default_config, str(toml_file))
+
+    # Verify generic LLM configuration remains unchanged
+    generic_llm = default_config.get_llm_config('llm')
+    assert generic_llm.model == 'base-model'
+    assert generic_llm.api_key == 'base-api-key'
+    assert generic_llm.embedding_model == 'base-embedding'
+    assert generic_llm.num_retries == 3
+
+    # Verify custom_full LLM overrides all attributes
+    custom_full = default_config.get_llm_config('custom_full')
+    assert custom_full.model == 'full-custom-model'
+    assert custom_full.api_key == 'full-custom-api-key'
+    assert custom_full.embedding_model == 'full-custom-embedding'
+    assert custom_full.num_retries == 10  # overridden value
+
+
+def test_load_from_toml_llm_custom_partial_override(
+    default_config: AppConfig, generic_llm_toml: str
+) -> None:
+    """Test that custom LLM configurations can partially override attributes
+    from the generic [llm] section while inheriting others.
+    """
+    load_from_toml(default_config, generic_llm_toml)
+
+    # Verify custom1 LLM overrides 'model' and 'api_key' but inherits 'num_retries'
+    custom1 = default_config.get_llm_config('custom1')
+    assert custom1.model == 'custom-model-1'
+    assert custom1.api_key == 'custom-api-key-1'
+    assert custom1.embedding_model == 'base-embedding'
+    assert custom1.num_retries == 3  # from [llm]
+
+    # Verify custom2 LLM overrides 'model', 'api_key', and 'num_retries'
+    custom2 = default_config.get_llm_config('custom2')
+    assert custom2.model == 'custom-model-2'
+    assert custom2.api_key == 'custom-api-key-2'
+    assert custom2.embedding_model == 'base-embedding'
+    assert custom2.num_retries == 5  # Overridden value
+
+
+def test_load_from_toml_llm_custom_no_override(
+    default_config: AppConfig, generic_llm_toml: str
+) -> None:
+    """Test that custom LLM configurations with no additional overrides
+    inherit all non-specified attributes from the generic [llm] section.
+    """
+    load_from_toml(default_config, generic_llm_toml)
+
+    # Verify custom3 LLM inherits 'embedding_model' and 'num_retries' from generic
+    custom3 = default_config.get_llm_config('custom3')
+    assert custom3.model == 'custom-model-3'
+    assert custom3.api_key == 'custom-api-key-3'
+    assert custom3.embedding_model == 'base-embedding'
+    assert custom3.num_retries == 3  # from [llm]
+
+
+def test_load_from_toml_llm_missing_generic(
+    default_config: AppConfig, tmp_path: pathlib.Path
+) -> None:
+    """Test that custom LLM configurations without a generic [llm] section
+    use only their own attributes and fallback to defaults for others.
+    """
+    toml_content = """
+[core]
+workspace_base = "./workspace"
+
+[llm.custom_only]
+model = "custom-only-model"
+api_key = "custom-only-api-key"
+    """
+    toml_file = tmp_path / 'custom_only_llm.toml'
+    toml_file.write_text(toml_content)
+
+    load_from_toml(default_config, str(toml_file))
+
+    # Verify custom_only LLM uses its own attributes and defaults for others
+    custom_only = default_config.get_llm_config('custom_only')
+    assert custom_only.model == 'custom-only-model'
+    assert custom_only.api_key == 'custom-only-api-key'
+    assert custom_only.embedding_model == 'local'  # default value
+    assert custom_only.num_retries == 8  # default value
+
+
+def test_load_from_toml_llm_invalid_config(
+    default_config: AppConfig, tmp_path: pathlib.Path
+) -> None:
+    """Test that invalid custom LLM configurations do not override the generic
+    and raise appropriate warnings.
+    """
+    toml_content = """
+[core]
+workspace_base = "./workspace"
+
+[llm]
+model = "base-model"
+api_key = "base-api-key"
+num_retries = 3
+
+[llm.invalid_custom]
+unknown_attr = "should_not_exist"
+    """
+    toml_file = tmp_path / 'invalid_custom_llm.toml'
+    toml_file.write_text(toml_content)
+
+    load_from_toml(default_config, str(toml_file))
+
+    # Verify generic LLM is loaded correctly
+    generic_llm = default_config.get_llm_config('llm')
+    assert generic_llm.model == 'base-model'
+    assert generic_llm.api_key == 'base-api-key'
+    assert generic_llm.num_retries == 3
+
+    # Verify invalid_custom LLM does not override generic attributes
+    custom_invalid = default_config.get_llm_config('invalid_custom')
+    assert custom_invalid.model == 'base-model'
+    assert custom_invalid.api_key == 'base-api-key'
+    assert custom_invalid.num_retries == 3  # default value
+    assert custom_invalid.embedding_model == 'local'  # default value

From 3d1cd8349bc745026b5a04ae00a361ede8cc41e8 Mon Sep 17 00:00:00 2001
From: openhands <openhands@all-hands.dev>
Date: Sat, 14 Dec 2024 07:36:47 +0000
Subject: [PATCH 03/11] Fix pr #4415: Fix llm_config fallback

---
 docs/modules/usage/configuration-options.md   |   4 +
 docs/modules/usage/llms/custom-llm-configs.md | 109 ++++++++++++++++++
 2 files changed, 113 insertions(+)
 create mode 100644 docs/modules/usage/llms/custom-llm-configs.md

diff --git a/docs/modules/usage/configuration-options.md b/docs/modules/usage/configuration-options.md
index 7a2718d7d786..8acf770834b7 100644
--- a/docs/modules/usage/configuration-options.md
+++ b/docs/modules/usage/configuration-options.md
@@ -182,6 +182,10 @@ The LLM (Large Language Model) configuration options are defined in the `[llm]`
 
 To use these with the docker command, pass in `-e LLM_<option>`. Example: `-e LLM_NUM_RETRIES`.
 
+:::note
+For development setups, you can also define custom named LLM configurations. See [Custom LLM Configurations](./llms/custom-llm-configs.md) for details.
+:::
+
 **AWS Credentials**
 - `aws_access_key_id`
   - Type: `str`
diff --git a/docs/modules/usage/llms/custom-llm-configs.md b/docs/modules/usage/llms/custom-llm-configs.md
new file mode 100644
index 000000000000..fb1656436351
--- /dev/null
+++ b/docs/modules/usage/llms/custom-llm-configs.md
@@ -0,0 +1,109 @@
+# Custom LLM Configurations
+
+OpenHands supports defining multiple named LLM configurations in your `config.toml` file. This feature allows you to use different LLM configurations for different purposes, such as using a cheaper model for tasks that don't require high-quality responses, or using different models with different parameters for specific agents.
+
+## How It Works
+
+Named LLM configurations are defined in the `config.toml` file using sections that start with `llm.`. For example:
+
+```toml
+# Default LLM configuration
+[llm]
+model = "gpt-4"
+api_key = "your-api-key"
+temperature = 0.0
+
+# Custom LLM configuration for a cheaper model
+[llm.gpt3]
+model = "gpt-3.5-turbo"
+api_key = "your-api-key"
+temperature = 0.2
+
+# Another custom configuration with different parameters
+[llm.high-creativity]
+model = "gpt-4"
+api_key = "your-api-key"
+temperature = 0.8
+top_p = 0.9
+```
+
+Each named configuration inherits all settings from the default `[llm]` section and can override any of those settings. You can define as many custom configurations as needed.
+
+## Using Custom Configurations
+
+### With Agents
+
+You can specify which LLM configuration an agent should use by setting the `llm_config` parameter in the agent's configuration section:
+
+```toml
+[agent.RepoExplorerAgent]
+# Use the cheaper GPT-3 configuration for this agent
+llm_config = 'gpt3'
+
+[agent.CodeWriterAgent]
+# Use the high creativity configuration for this agent
+llm_config = 'high-creativity'
+```
+
+### Configuration Options
+
+Each named LLM configuration supports all the same options as the default LLM configuration. These include:
+
+- Model selection (`model`)
+- API configuration (`api_key`, `base_url`, etc.)
+- Model parameters (`temperature`, `top_p`, etc.)
+- Retry settings (`num_retries`, `retry_multiplier`, etc.)
+- Token limits (`max_input_tokens`, `max_output_tokens`)
+- And all other LLM configuration options
+
+For a complete list of available options, see the [LLM Configuration](../configuration-options.md#llm-configuration) section in the Configuration Options documentation.
+
+## Use Cases
+
+Custom LLM configurations are particularly useful in several scenarios:
+
+1. **Cost Optimization**: Use cheaper models for tasks that don't require high-quality responses, like repository exploration or simple file operations.
+
+2. **Task-Specific Tuning**: Configure different temperature and top_p values for tasks that require different levels of creativity or determinism.
+
+3. **Different Providers**: Use different LLM providers or API endpoints for different tasks.
+
+4. **Testing and Development**: Easily switch between different model configurations during development and testing.
+
+## Example: Cost Optimization
+
+Here's a practical example of using custom LLM configurations to optimize costs:
+
+```toml
+# Default configuration using GPT-4 for high-quality responses
+[llm]
+model = "gpt-4"
+api_key = "your-api-key"
+temperature = 0.0
+
+# Cheaper configuration for repository exploration
+[llm.repo-explorer]
+model = "gpt-3.5-turbo"
+temperature = 0.2
+
+# Configuration for code generation
+[llm.code-gen]
+model = "gpt-4"
+temperature = 0.0
+max_output_tokens = 2000
+
+[agent.RepoExplorerAgent]
+llm_config = 'repo-explorer'
+
+[agent.CodeWriterAgent]
+llm_config = 'code-gen'
+```
+
+In this example:
+- Repository exploration uses a cheaper model since it mainly involves understanding and navigating code
+- Code generation uses GPT-4 with a higher token limit for generating larger code blocks
+- The default configuration remains available for other tasks
+
+:::note
+Custom LLM configurations are only available when using OpenHands in development mode. For production deployments, use the standard configuration options.
+:::

From bf8feff14008d9212618f098cdddd00ee9b8cc48 Mon Sep 17 00:00:00 2001
From: openhands <openhands@all-hands.dev>
Date: Sat, 14 Dec 2024 13:46:27 +0000
Subject: [PATCH 04/11] fix: Handle draft_editor and fallback values in LLM
 configs

- Adds proper fallback mechanism from generic LLM config to custom configs
- Adds special handling for draft_editor field:
  - Falls back to generic config value if not specified
  - Can be set to None using 'null' in TOML
  - Can be overridden with custom value
---
 openhands/core/config/llm_config.py | 19 ++++--
 openhands/core/config/utils.py      | 37 ++++++++++--
 tests/unit/test_llm_draft_config.py | 92 +++++++++++++++++++++++++++++
 3 files changed, 138 insertions(+), 10 deletions(-)
 create mode 100644 tests/unit/test_llm_draft_config.py

diff --git a/openhands/core/config/llm_config.py b/openhands/core/config/llm_config.py
index 477b47ccdbe1..84a6e1e95fc2 100644
--- a/openhands/core/config/llm_config.py
+++ b/openhands/core/config/llm_config.py
@@ -130,8 +130,19 @@ def from_dict(cls, llm_config_dict: dict) -> 'LLMConfig':
         This function is used to create an LLMConfig object from a dictionary,
         with the exception of the 'draft_editor' key, which is a nested LLMConfig object.
         """
-        args = {k: v for k, v in llm_config_dict.items() if not isinstance(v, dict)}
-        if 'draft_editor' in llm_config_dict:
-            draft_editor_config = LLMConfig(**llm_config_dict['draft_editor'])
-            args['draft_editor'] = draft_editor_config
+        # Keep None values to preserve defaults, filter out other dicts
+        args = {
+            k: v
+            for k, v in llm_config_dict.items()
+            if not isinstance(v, dict) or v is None
+        }
+        if (
+            'draft_editor' in llm_config_dict
+            and llm_config_dict['draft_editor'] is not None
+        ):
+            if isinstance(llm_config_dict['draft_editor'], LLMConfig):
+                args['draft_editor'] = llm_config_dict['draft_editor']
+            else:
+                draft_editor_config = LLMConfig(**llm_config_dict['draft_editor'])
+                args['draft_editor'] = draft_editor_config
         return cls(**args)
diff --git a/openhands/core/config/utils.py b/openhands/core/config/utils.py
index 2b985c4f5937..e947bddbbc81 100644
--- a/openhands/core/config/utils.py
+++ b/openhands/core/config/utils.py
@@ -140,18 +140,28 @@ def load_from_toml(cfg: AppConfig, toml_file: str = 'config.toml'):
                     logger.openhands_logger.debug(
                         'Attempt to load default LLM config from config toml'
                     )
-                    # Extract generic LLM fields
-                    generic_llm_fields = {
-                        k: v for k, v in value.items() if not isinstance(v, dict)
-                    }
+                    # Extract generic LLM fields, keeping draft_editor
+                    generic_llm_fields = {}
+                    for k, v in value.items():
+                        if not isinstance(v, dict) or k == 'draft_editor':
+                            generic_llm_fields[k] = v
+                    logger.openhands_logger.debug(
+                        f'Generic LLM fields: {generic_llm_fields}'
+                    )
                     generic_llm_config = LLMConfig.from_dict(generic_llm_fields)
+                    logger.openhands_logger.debug(
+                        f'Generic LLM config dict: {generic_llm_config.__dict__}'
+                    )
                     cfg.set_llm_config(generic_llm_config, 'llm')
 
                     # Process custom named LLM configs
                     for nested_key, nested_value in value.items():
                         if isinstance(nested_value, dict):
                             logger.openhands_logger.debug(
-                                f'Attempt to load group {nested_key} from config toml as LLM config'
+                                f'Processing custom LLM config "{nested_key}":'
+                            )
+                            logger.openhands_logger.debug(
+                                f'  Nested value: {nested_value}'
                             )
                             # Apply generic LLM config with custom LLM overrides, e.g.
                             # [llm]
@@ -160,8 +170,23 @@ def load_from_toml(cfg: AppConfig, toml_file: str = 'config.toml'):
                             # [llm.claude]
                             # model="claude-3-5-sonnet"
                             # results in num_retries APPLIED to claude-3-5-sonnet
+                            custom_fields = {}
+                            for k, v in nested_value.items():
+                                if not isinstance(v, dict) or k == 'draft_editor':
+                                    custom_fields[k] = v
                             merged_llm_dict = generic_llm_config.__dict__.copy()
-                            merged_llm_dict.update(nested_value)
+                            merged_llm_dict.update(custom_fields)
+                            # Handle draft_editor with fallback values:
+                            # - If draft_editor is "null", use None
+                            # - If draft_editor is in custom fields, use that value
+                            # - If draft_editor is not specified, fall back to generic config value
+                            if 'draft_editor' in custom_fields:
+                                if custom_fields['draft_editor'] == 'null':
+                                    merged_llm_dict['draft_editor'] = None
+                            else:
+                                merged_llm_dict['draft_editor'] = (
+                                    generic_llm_config.draft_editor
+                                )
                             custom_llm_config = LLMConfig.from_dict(merged_llm_dict)
                             cfg.set_llm_config(custom_llm_config, nested_key)
                 elif key is not None and key.lower() == 'security':
diff --git a/tests/unit/test_llm_draft_config.py b/tests/unit/test_llm_draft_config.py
new file mode 100644
index 000000000000..160bbf594899
--- /dev/null
+++ b/tests/unit/test_llm_draft_config.py
@@ -0,0 +1,92 @@
+import pathlib
+
+import pytest
+
+from openhands.core.config import AppConfig
+from openhands.core.config.utils import load_from_toml
+
+
+@pytest.fixture
+def draft_llm_toml(tmp_path: pathlib.Path) -> str:
+    toml_content = """
+[core]
+workspace_base = "./workspace"
+
+[llm]
+model = "base-model"
+api_key = "base-api-key"
+draft_editor = { model = "draft-model", api_key = "draft-api-key" }
+
+[llm.custom1]
+model = "custom-model-1"
+api_key = "custom-api-key-1"
+# Should use draft_editor from [llm] as fallback
+
+[llm.custom2]
+model = "custom-model-2"
+api_key = "custom-api-key-2"
+draft_editor = { model = "custom-draft", api_key = "custom-draft-key" }
+
+[llm.custom3]
+model = "custom-model-3"
+api_key = "custom-api-key-3"
+draft_editor = "null"  # Explicitly set to null in TOML
+    """
+    toml_file = tmp_path / 'llm_config.toml'
+    toml_file.write_text(toml_content)
+    return str(toml_file)
+
+
+def test_draft_editor_fallback(draft_llm_toml):
+    """Test that draft_editor is correctly handled in different scenarios:
+    - Falls back to generic [llm] section value
+    - Uses custom value when specified
+    - Can be explicitly set to null
+    """
+    config = AppConfig()
+
+    # Verify default draft_editor is None
+    default_llm = config.get_llm_config('llm')
+    assert default_llm.draft_editor is None
+
+    # Load config from TOML
+    load_from_toml(config, draft_llm_toml)
+
+    # Verify generic LLM draft_editor
+    generic_llm = config.get_llm_config('llm')
+    assert generic_llm.draft_editor is not None
+    assert generic_llm.draft_editor.model == 'draft-model'
+    assert generic_llm.draft_editor.api_key == 'draft-api-key'
+
+    # Verify custom1 uses draft_editor from generic as fallback
+    custom1 = config.get_llm_config('custom1')
+    assert custom1.model == 'custom-model-1'
+    assert custom1.draft_editor is not None
+    assert custom1.draft_editor.model == 'draft-model'
+    assert custom1.draft_editor.api_key == 'draft-api-key'
+
+    # Verify custom2 overrides draft_editor
+    custom2 = config.get_llm_config('custom2')
+    assert custom2.model == 'custom-model-2'
+    assert custom2.draft_editor is not None
+    assert custom2.draft_editor.model == 'custom-draft'
+    assert custom2.draft_editor.api_key == 'custom-draft-key'
+
+    # Verify custom3 has draft_editor explicitly set to None
+    custom3 = config.get_llm_config('custom3')
+    assert custom3.model == 'custom-model-3'
+    assert custom3.draft_editor is None
+
+
+def test_draft_editor_defaults(draft_llm_toml):
+    """Test that draft_editor uses default values from LLMConfig when not specified"""
+    config = AppConfig()
+    load_from_toml(config, draft_llm_toml)
+
+    generic_llm = config.get_llm_config('llm')
+    assert generic_llm.draft_editor.num_retries == 8  # Default from LLMConfig
+    assert generic_llm.draft_editor.embedding_model == 'local'  # Default from LLMConfig
+
+    custom2 = config.get_llm_config('custom2')
+    assert custom2.draft_editor.num_retries == 8  # Default from LLMConfig
+    assert custom2.draft_editor.embedding_model == 'local'  # Default from LLMConfig

From beab343366babc10e0462b15110c74be1ff92b0c Mon Sep 17 00:00:00 2001
From: Engel Nyst <enyst@users.noreply.github.com>
Date: Thu, 2 Jan 2025 22:13:35 +0100
Subject: [PATCH 05/11] fix formatting

---
 docs/modules/usage/llms/custom-llm-configs.md | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/docs/modules/usage/llms/custom-llm-configs.md b/docs/modules/usage/llms/custom-llm-configs.md
index fb1656436351..3e2b5632caed 100644
--- a/docs/modules/usage/llms/custom-llm-configs.md
+++ b/docs/modules/usage/llms/custom-llm-configs.md
@@ -62,17 +62,14 @@ For a complete list of available options, see the [LLM Configuration](../configu
 
 Custom LLM configurations are particularly useful in several scenarios:
 
-1. **Cost Optimization**: Use cheaper models for tasks that don't require high-quality responses, like repository exploration or simple file operations.
-
-2. **Task-Specific Tuning**: Configure different temperature and top_p values for tasks that require different levels of creativity or determinism.
-
-3. **Different Providers**: Use different LLM providers or API endpoints for different tasks.
-
-4. **Testing and Development**: Easily switch between different model configurations during development and testing.
+- **Cost Optimization**: Use cheaper models for tasks that don't require high-quality responses, like repository exploration or simple file operations.
+- **Task-Specific Tuning**: Configure different temperature and top_p values for tasks that require different levels of creativity or determinism.
+- **Different Providers**: Use different LLM providers or API endpoints for different tasks.
+- **Testing and Development**: Easily switch between different model configurations during development and testing.
 
 ## Example: Cost Optimization
 
-Here's a practical example of using custom LLM configurations to optimize costs:
+A practical example of using custom LLM configurations to optimize costs:
 
 ```toml
 # Default configuration using GPT-4 for high-quality responses
@@ -105,5 +102,5 @@ In this example:
 - The default configuration remains available for other tasks
 
 :::note
-Custom LLM configurations are only available when using OpenHands in development mode. For production deployments, use the standard configuration options.
+Custom LLM configurations are only available when using OpenHands in development mode, via `main.py` or `cli.py`. When running via `docker run`, please use the standard configuration options.
 :::

From df74bd4e417dbb59d4656709c65ca3d499da4eef Mon Sep 17 00:00:00 2001
From: Engel Nyst <enyst@users.noreply.github.com>
Date: Tue, 14 Jan 2025 23:29:21 +0100
Subject: [PATCH 06/11] remove potentially sensitive logs

---
 openhands/core/config/utils.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/openhands/core/config/utils.py b/openhands/core/config/utils.py
index 3932cd46a49f..8beeeaf26727 100644
--- a/openhands/core/config/utils.py
+++ b/openhands/core/config/utils.py
@@ -150,13 +150,7 @@ def load_from_toml(cfg: AppConfig, toml_file: str = 'config.toml'):
                     for k, v in value.items():
                         if not isinstance(v, dict) or k == 'draft_editor':
                             generic_llm_fields[k] = v
-                    logger.openhands_logger.debug(
-                        f'Generic LLM fields: {generic_llm_fields}'
-                    )
                     generic_llm_config = LLMConfig.from_dict(generic_llm_fields)
-                    logger.openhands_logger.debug(
-                        f'Generic LLM config dict: {generic_llm_config.__dict__}'
-                    )
                     cfg.set_llm_config(generic_llm_config, 'llm')
 
                     # Process custom named LLM configs
@@ -165,9 +159,6 @@ def load_from_toml(cfg: AppConfig, toml_file: str = 'config.toml'):
                             logger.openhands_logger.debug(
                                 f'Processing custom LLM config "{nested_key}":'
                             )
-                            logger.openhands_logger.debug(
-                                f'  Nested value: {nested_value}'
-                            )
                             # Apply generic LLM config with custom LLM overrides, e.g.
                             # [llm]
                             # model="..."

From 4b662363ceeadb58c06630224957df4a8410cc76 Mon Sep 17 00:00:00 2001
From: Engel Nyst <enyst@users.noreply.github.com>
Date: Tue, 14 Jan 2025 23:37:18 +0100
Subject: [PATCH 07/11] Update openhands/core/config/utils.py

---
 openhands/core/config/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/openhands/core/config/utils.py b/openhands/core/config/utils.py
index da5c18901418..3cd503a70aaf 100644
--- a/openhands/core/config/utils.py
+++ b/openhands/core/config/utils.py
@@ -144,6 +144,7 @@ def load_from_toml(cfg: AppConfig, toml_file: str = 'config.toml'):
                     logger.openhands_logger.debug(
                         'Attempt to load default LLM config from config toml'
                     )
+                    # TODO clean up draft_editor
                     # Extract generic LLM fields, keeping draft_editor
                     generic_llm_fields = {}
                     for k, v in value.items():

From e4076ef613bac141542cca3523dcda9cd9099907 Mon Sep 17 00:00:00 2001
From: Engel Nyst <enyst@users.noreply.github.com>
Date: Tue, 14 Jan 2025 23:38:49 +0100
Subject: [PATCH 08/11] Update openhands/core/config/utils.py

---
 openhands/core/config/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/openhands/core/config/utils.py b/openhands/core/config/utils.py
index 3cd503a70aaf..676b38252143 100644
--- a/openhands/core/config/utils.py
+++ b/openhands/core/config/utils.py
@@ -172,6 +172,7 @@ def load_from_toml(cfg: AppConfig, toml_file: str = 'config.toml'):
                                     custom_fields[k] = v
                             merged_llm_dict = generic_llm_config.__dict__.copy()
                             merged_llm_dict.update(custom_fields)
+                            # TODO clean up draft_editor
                             # Handle draft_editor with fallback values:
                             # - If draft_editor is "null", use None
                             # - If draft_editor is in custom fields, use that value

From 7a90f25d6d5fa55e239116363141748539efa690 Mon Sep 17 00:00:00 2001
From: Engel Nyst <enyst@users.noreply.github.com>
Date: Wed, 15 Jan 2025 01:10:32 +0100
Subject: [PATCH 09/11] fix use fallback

---
 openhands/core/config/utils.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/openhands/core/config/utils.py b/openhands/core/config/utils.py
index 676b38252143..82fbfe1e2601 100644
--- a/openhands/core/config/utils.py
+++ b/openhands/core/config/utils.py
@@ -491,7 +491,11 @@ def setup_config_from_args(args: argparse.Namespace) -> AppConfig:
 
     # Override with command line arguments if provided
     if args.llm_config:
-        llm_config = get_llm_config_arg(args.llm_config)
+        # if we didn't already load it, get it from the toml file
+        if args.llm_config not in config.llms:
+            llm_config = get_llm_config_arg(args.llm_config)
+        else:
+            llm_config = config.llms[args.llm_config]
         if llm_config is None:
             raise ValueError(f'Invalid toml file, cannot read {args.llm_config}')
         config.set_llm_config(llm_config)

From cb7126df01ec105f3bf0615206cf0e27fb21e54d Mon Sep 17 00:00:00 2001
From: Engel Nyst <enyst@users.noreply.github.com>
Date: Wed, 15 Jan 2025 01:14:54 +0100
Subject: [PATCH 10/11] fix doc link

---
 docs/modules/usage/configuration-options.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/modules/usage/configuration-options.md b/docs/modules/usage/configuration-options.md
index ee7c3f4d5af7..b2fd2d663ae9 100644
--- a/docs/modules/usage/configuration-options.md
+++ b/docs/modules/usage/configuration-options.md
@@ -141,7 +141,7 @@ The LLM (Large Language Model) configuration options are defined in the `[llm]`
 To use these with the docker command, pass in `-e LLM_<option>`. Example: `-e LLM_NUM_RETRIES`.
 
 :::note
-For development setups, you can also define custom named LLM configurations. See [Custom LLM Configurations](./llms/custom-llm-configs.md) for details.
+For development setups, you can also define custom named LLM configurations. See [Custom LLM Configurations](https://docs.all-hands.dev/modules/usage/llms/custom-llm-configs) for details.
 :::
 
 **AWS Credentials**

From 3e5f40e1c14188ceb5594c4d9dddb7588a1b7dab Mon Sep 17 00:00:00 2001
From: Engel Nyst <enyst@users.noreply.github.com>
Date: Wed, 15 Jan 2025 01:55:51 +0100
Subject: [PATCH 11/11] fix doc link

---
 docs/modules/usage/llms/custom-llm-configs.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/modules/usage/llms/custom-llm-configs.md b/docs/modules/usage/llms/custom-llm-configs.md
index 3e2b5632caed..45ba11352824 100644
--- a/docs/modules/usage/llms/custom-llm-configs.md
+++ b/docs/modules/usage/llms/custom-llm-configs.md
@@ -56,7 +56,7 @@ Each named LLM configuration supports all the same options as the default LLM co
 - Token limits (`max_input_tokens`, `max_output_tokens`)
 - And all other LLM configuration options
 
-For a complete list of available options, see the [LLM Configuration](../configuration-options.md#llm-configuration) section in the Configuration Options documentation.
+For a complete list of available options, see the LLM Configuration section in the [Configuration Options](../configuration-options.md) documentation.
 
 ## Use Cases