diff --git a/berkeley-function-call-leaderboard/bfcl/__main__.py b/berkeley-function-call-leaderboard/bfcl/__main__.py index deaf7d4ae..b77fade3a 100644 --- a/berkeley-function-call-leaderboard/bfcl/__main__.py +++ b/berkeley-function-call-leaderboard/bfcl/__main__.py @@ -2,6 +2,8 @@ from collections import namedtuple from datetime import datetime from typing import List +import json +import os import typer from bfcl._llm_response_generation import main as generation_main @@ -10,6 +12,7 @@ from bfcl.model_handler.handler_map import HANDLER_MAP from dotenv import load_dotenv from tabulate import tabulate +from bfcl.model_handler.handler_loader import HandlerLoader class ExecutionOrderGroup(typer.core.TyperGroup): @@ -52,8 +55,20 @@ def models(): """ List available models. """ + available_models = set(HANDLER_MAP.keys()) + + # If a custom handler setting exists, add it to the + handler_config_path = os.getenv("BFCL_HANDLER_CONFIG") + if handler_config_path and os.path.exists(handler_config_path): + try: + with open(handler_config_path) as f: + handler_config = json.load(f) + available_models.update(handler_config.keys()) + except Exception as e: + print(f"Warning: Error loading custom handler config: {str(e)}") + table = tabulate( - [[model] for model in HANDLER_MAP.keys()], + [[model] for model in sorted(available_models)], tablefmt="plain", colalign=("left",), ) diff --git a/berkeley-function-call-leaderboard/bfcl/_llm_response_generation.py b/berkeley-function-call-leaderboard/bfcl/_llm_response_generation.py index 6122a6d9a..93d605878 100644 --- a/berkeley-function-call-leaderboard/bfcl/_llm_response_generation.py +++ b/berkeley-function-call-leaderboard/bfcl/_llm_response_generation.py @@ -15,7 +15,7 @@ TEST_FILE_MAPPING, ) from bfcl.eval_checker.eval_runner_helper import load_file -from bfcl.model_handler.handler_map import HANDLER_MAP +from bfcl.model_handler.handler_loader import HandlerLoader from bfcl.model_handler.model_style import ModelStyle from bfcl.utils import is_executable, is_multi_turn from tqdm import tqdm @@ -49,8 +49,12 @@ def get_args(): def build_handler(model_name, temperature): - handler = HANDLER_MAP[model_name](model_name, temperature) - return handler + """Create a handler instance""" + handler_class = HandlerLoader.get_handler_class(model_name) + if handler_class is None: + raise ValueError(f"No handler found for model: {model_name}") + + return handler_class(model_name, temperature) def sort_key(entry): diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/handler_loader.py b/berkeley-function-call-leaderboard/bfcl/model_handler/handler_loader.py new file mode 100644 index 000000000..085bae7e8 --- /dev/null +++ b/berkeley-function-call-leaderboard/bfcl/model_handler/handler_loader.py @@ -0,0 +1,61 @@ +import json +import importlib.util +import os +from pathlib import Path +from typing import Type, Optional + +from bfcl.model_handler.base_handler import BaseHandler +from bfcl.model_handler.handler_map import HANDLER_MAP + +class HandlerLoader: + @staticmethod + def load_handler_class(module_path: str, class_name: str) -> Optional[Type[BaseHandler]]: + """Dynamically load handler classes from a specified path""" + try: + abs_path = str(Path(module_path).resolve()) + spec = importlib.util.spec_from_file_location("custom_module", abs_path) + if spec is None or spec.loader is None: + raise ImportError(f"Could not load spec for module: {module_path}") + + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + handler_class = getattr(module, class_name, None) + if handler_class is None: + raise AttributeError(f"Class {class_name} not found in {module_path}") + + # Checking for BaseHandler Inheritance + if not issubclass(handler_class, BaseHandler): + raise TypeError(f"Class {class_name} must inherit from BaseHandler") + + return handler_class + + except Exception as e: + print(f"Error loading handler class {class_name} from {module_path}: {str(e)}") + return None + + @staticmethod + def get_handler_class(model_name: str) -> Optional[Type[BaseHandler]]: + """Returns the handler class corresponding to the model name""" + # Check the path to the handler mapping file in an environment variable + handler_config_path = os.getenv("BFCL_HANDLER_CONFIG") + + if handler_config_path and os.path.exists(handler_config_path): + try: + with open(handler_config_path) as f: + handler_config = json.load(f) + + if model_name in handler_config: + config = handler_config[model_name] + handler_class = HandlerLoader.load_handler_class( + config["module_path"], + config["class_name"] + ) + if handler_class: + return handler_class + + except Exception as e: + print(f"Error loading custom handler config: {str(e)}") + + # Lookup in the default handler map + return HANDLER_MAP.get(model_name)