Use litellm's modify params (#5636)

config.template.toml

@@ -154,6 +154,10 @@ model = "gpt-4o"
 # Drop any unmapped (unsupported) params without causing an exception
 #drop_params = false
 
+# Modify params for litellm to do transformations like adding a default message, when a message is empty.
+# Note: this setting is global, unlike drop_params, it cannot be overridden in each call to litellm.
+#modify_params = true
+
 # Using the prompt caching feature if provided by the LLM and supported
 #caching_prompt = true
 

evaluation/benchmarks/EDA/run_infer.py

@@ -202,6 +202,9 @@ def process_instance(
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
+
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

evaluation/benchmarks/agent_bench/run_infer.py

@@ -307,6 +307,8 @@ def process_instance(
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
 
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')

evaluation/benchmarks/aider_bench/run_infer.py

@@ -279,6 +279,8 @@ def process_instance(
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
 
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')

evaluation/benchmarks/biocoder/run_infer.py

@@ -328,6 +328,8 @@ def process_instance(
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
 
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')

evaluation/benchmarks/bird/run_infer.py

@@ -456,6 +456,8 @@ def execute_sql(db_path, sql):
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

evaluation/benchmarks/browsing_delegation/run_infer.py

@@ -142,6 +142,8 @@ def process_instance(
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
 
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')

evaluation/benchmarks/commit0_bench/run_infer.py

@@ -571,6 +571,8 @@ def commit0_setup(dataset: pd.DataFrame, repo_split: str) -> pd.DataFrame:
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
         llm_config.log_completions = True
 
     if llm_config is None:

evaluation/benchmarks/discoverybench/run_infer.py

@@ -466,6 +466,8 @@ def create_dataset(repo_location: str, split: str = 'test'):
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

evaluation/benchmarks/gaia/run_infer.py

@@ -238,6 +238,9 @@ def process_instance(
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
+
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

evaluation/benchmarks/gorilla/run_infer.py

@@ -146,6 +146,8 @@ def process_instance(
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

evaluation/benchmarks/gpqa/run_infer.py

@@ -326,6 +326,9 @@ def process_instance(
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
+
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

evaluation/benchmarks/humanevalfix/run_infer.py

@@ -285,6 +285,8 @@ def process_instance(
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

evaluation/benchmarks/logic_reasoning/run_infer.py

@@ -288,6 +288,8 @@ def process_instance(
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

evaluation/benchmarks/miniwob/run_infer.py

@@ -231,6 +231,8 @@ def process_instance(
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

evaluation/benchmarks/mint/run_infer.py

@@ -279,6 +279,8 @@ def process_instance(
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

evaluation/benchmarks/ml_bench/run_analysis.py

@@ -124,6 +124,9 @@ def classify_error(llm: LLM, failed_case: dict) -> str:
     # for details of how to set `llm_config`
     if args.llm_config:
         specified_llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        specified_llm_config.modify_params = False
+
         if specified_llm_config:
             config.llm = specified_llm_config
     logger.info(f'Config for evaluation: {config}')

evaluation/benchmarks/ml_bench/run_infer.py

@@ -292,6 +292,8 @@ def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool =
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

evaluation/benchmarks/scienceagentbench/run_infer.py

@@ -272,6 +272,8 @@ def process_instance(
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

evaluation/benchmarks/swe_bench/run_infer.py

@@ -490,6 +490,8 @@ def filter_dataset(dataset: pd.DataFrame, filter_column: str) -> pd.DataFrame:
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
         llm_config.log_completions = True
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
 
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')

evaluation/benchmarks/toolqa/run_infer.py

@@ -181,6 +181,9 @@ def process_instance(instance: Any, metadata: EvalMetadata, reset_logger: bool =
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
+
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

evaluation/benchmarks/webarena/run_infer.py

@@ -212,6 +212,8 @@ def process_instance(
     llm_config = None
     if args.llm_config:
         llm_config = get_llm_config_arg(args.llm_config)
+        # modify_params must be False for evaluation purpose, for reproducibility and accurancy of results
+        llm_config.modify_params = False
     if llm_config is None:
         raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}')
 

openhands/core/config/llm_config.py

@@ -38,6 +38,7 @@ class LLMConfig:
         output_cost_per_token: The cost per output token. This will available in logs for the user to check.
         ollama_base_url: The base URL for the OLLAMA API.
         drop_params: Drop any unmapped (unsupported) params without causing an exception.
+        modify_params: Modify params allows litellm to do transformations like adding a default message, when a message is empty.
         disable_vision: If model is vision capable, this option allows to disable image processing (useful for cost reduction).
         caching_prompt: Use the prompt caching feature if provided by the LLM and supported by the provider.
         log_completions: Whether to log LLM completions to the state.
@@ -72,7 +73,10 @@ class LLMConfig:
     input_cost_per_token: float | None = None
     output_cost_per_token: float | None = None
     ollama_base_url: str | None = None
+    # This setting can be sent in each call to litellm
     drop_params: bool = True
+    # Note: this setting is actually global, unlike drop_params
+    modify_params: bool = True
     disable_vision: bool | None = None
     caching_prompt: bool = True
     log_completions: bool = False

openhands/llm/llm.py

@@ -101,7 +101,6 @@ def __init__(
         self.cost_metric_supported: bool = True
         self.config: LLMConfig = copy.deepcopy(config)
 
-        # litellm actually uses base Exception here for unknown model
         self.model_info: ModelInfo | None = None
 
         if self.config.log_completions:
@@ -206,6 +205,11 @@ def wrapper(*args, **kwargs):
                         'anthropic-beta': 'prompt-caching-2024-07-31',
                     }
 
+            # set litellm modify_params to the configured value
+            # True by default to allow litellm to do transformations like adding a default message, when a message is empty
+            # NOTE: this setting is global; unlike drop_params, it cannot be overridden in the litellm completion partial
+            litellm.modify_params = self.config.modify_params
+
             try:
                 # Record start time for latency measurement
                 start_time = time.time()