[BFCL] Introducing custom handlers

berkeley-function-call-leaderboard/bfcl/__main__.py

@@ -2,6 +2,8 @@
 from collections import namedtuple
 from datetime import datetime
 from typing import List
+import json
+import os
 
 import typer
 from bfcl._llm_response_generation import main as generation_main
@@ -10,6 +12,7 @@
 from bfcl.model_handler.handler_map import HANDLER_MAP
 from dotenv import load_dotenv
 from tabulate import tabulate
+from bfcl.model_handler.handler_loader import HandlerLoader
 
 
 class ExecutionOrderGroup(typer.core.TyperGroup):
@@ -52,8 +55,20 @@ def models():
     """
     List available models.
     """
+    available_models = set(HANDLER_MAP.keys())
+
+    # If a custom handler setting exists, add it to the
+    handler_config_path = os.getenv("BFCL_HANDLER_CONFIG")
+    if handler_config_path and os.path.exists(handler_config_path):
+        try:
+            with open(handler_config_path) as f:
+                handler_config = json.load(f)
+                available_models.update(handler_config.keys())
+        except Exception as e:
+            print(f"Warning: Error loading custom handler config: {str(e)}")
+
     table = tabulate(
-        [[model] for model in HANDLER_MAP.keys()],
+        [[model] for model in sorted(available_models)],
         tablefmt="plain",
         colalign=("left",),
     )

berkeley-function-call-leaderboard/bfcl/_llm_response_generation.py

@@ -15,7 +15,7 @@
     TEST_FILE_MAPPING,
 )
 from bfcl.eval_checker.eval_runner_helper import load_file
-from bfcl.model_handler.handler_map import HANDLER_MAP
+from bfcl.model_handler.handler_loader import HandlerLoader
 from bfcl.model_handler.model_style import ModelStyle
 from bfcl.utils import is_executable, is_multi_turn
 from tqdm import tqdm
@@ -49,8 +49,12 @@ def get_args():
 
 
 def build_handler(model_name, temperature):
-    handler = HANDLER_MAP[model_name](model_name, temperature)
-    return handler
+    """Create a handler instance"""
+    handler_class = HandlerLoader.get_handler_class(model_name)
+    if handler_class is None:
+        raise ValueError(f"No handler found for model: {model_name}")
+
+    return handler_class(model_name, temperature)
 
 
 def sort_key(entry):

berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner.py

@@ -23,6 +23,7 @@
 )
 from bfcl.eval_checker.multi_turn_eval.multi_turn_utils import is_empty_execute_response
 from bfcl.model_handler.handler_map import HANDLER_MAP
+from bfcl.model_handler.handler_loader import HandlerLoader
 from bfcl.utils import *
 from dotenv import load_dotenv
 from tqdm import tqdm
@@ -631,9 +632,12 @@ def main(model, test_category, api_sanity_check):
 
 
 def get_handler(model_name):
-    return HANDLER_MAP[model_name](
-        model_name, temperature=0
-    )  # Temperature doesn't matter for evaluation
+    """Create a handler instance"""
+    handler_class = HandlerLoader.get_handler_class(model_name)
+    if handler_class is None:
+        raise ValueError(f"No handler found for model: {model_name}")
+
+    return handler_class(model_name, temperature=0)
 
 
 if __name__ == "__main__":

berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner_helper.py

@@ -14,6 +14,7 @@
     write_list_of_dicts_to_file,
 )
 from tqdm import tqdm
+from bfcl.eval_checker.metadata_loader import metadata_loader
 
 
 def api_status_sanity_check_rest():
@@ -244,6 +245,9 @@ def get_cost_letency_info(model_name, cost_data, latency_data):
 def generate_leaderboard_csv(
     leaderboard_table, output_path, eval_models=None, eval_categories=None
 ):
+    # Load metadata at the beginning of the function
+    model_metadata, _, _ = metadata_loader.load_metadata()
+
     print("📈 Aggregating data to generate leaderboard score table...")
     data_non_live = []
     data_live = []
@@ -324,7 +328,7 @@ def generate_leaderboard_csv(
         data_non_live.append(
             [
                 "N/A",
-                MODEL_METADATA_MAPPING[model_name_escaped][0],
+                model_metadata[model_name_escaped][0],
                 overall_accuracy_non_live["accuracy"],
                 summary_ast_non_live["accuracy"],
                 summary_exec_non_live["accuracy"],
@@ -385,7 +389,7 @@ def generate_leaderboard_csv(
         data_live.append(
             [
                 "N/A",
-                MODEL_METADATA_MAPPING[model_name_escaped][0],
+                model_metadata[model_name_escaped][0],
                 overall_accuracy_live["accuracy"],
                 summary_ast_live["accuracy"],
                 python_simple_ast_live["accuracy"],
@@ -424,7 +428,7 @@ def generate_leaderboard_csv(
         data_multi_turn.append(
             [
                 "N/A",
-                MODEL_METADATA_MAPPING[model_name_escaped][0],
+                model_metadata[model_name_escaped][0],
                 overall_accuracy_multi_turn["accuracy"],
                 multi_turn_base["accuracy"],
                 multi_turn_miss_func["accuracy"],
@@ -451,8 +455,8 @@ def generate_leaderboard_csv(
             [
                 "N/A",
                 total_overall_accuracy["accuracy"],
-                MODEL_METADATA_MAPPING[model_name_escaped][0],
-                MODEL_METADATA_MAPPING[model_name_escaped][1],
+                model_metadata[model_name_escaped][0],
+                model_metadata[model_name_escaped][1],
                 cost,
                 latency_mean,
                 latency_std,
@@ -481,8 +485,8 @@ def generate_leaderboard_csv(
                 # multi_turn_composite["accuracy"],
                 total_relevance["accuracy"],
                 total_irrelevance["accuracy"],
-                MODEL_METADATA_MAPPING[model_name_escaped][2],
-                MODEL_METADATA_MAPPING[model_name_escaped][3],
+                model_metadata[model_name_escaped][2],
+                model_metadata[model_name_escaped][3],
             ]
         )
 
@@ -554,6 +558,7 @@ def generate_leaderboard_csv(
 
 def check_model_category_status(score_path):
     result_path = score_path.replace("score", "result")
+    model_metadata, _, _ = metadata_loader.load_metadata()
 
     leaderboard_categories = [
         "exec_simple",
@@ -578,8 +583,8 @@ def check_model_category_status(score_path):
 
     category_status = {}
 
-    # Check for all models in MODEL_METADATA_MAPPING
-    for model_name in MODEL_METADATA_MAPPING.keys():
+    # Check for all models in metadata
+    for model_name in model_metadata.keys():
         category_status[model_name] = {
             category: {"generated": False, "evaluated": False}
             for category in leaderboard_categories

berkeley-function-call-leaderboard/bfcl/eval_checker/metadata_loader.py

@@ -0,0 +1,54 @@
+import json
+import os
+from typing import Dict, Any
+
+from bfcl.model_handler.handler_map import local_inference_handler_map
+from bfcl.eval_checker.model_metadata import (
+    MODEL_METADATA_MAPPING,
+    OUTPUT_PRICE_PER_MILLION_TOKEN,
+    NO_COST_MODELS,
+)
+
+class MetadataLoader:
+    @staticmethod
+    def load_metadata() -> tuple[Dict[str, Any], Dict[str, float], list[str]]:
+        """
+        Load model metadata, pricing information, and list of no-cost models.
+
+        Returns:
+            tuple containing:
+            - metadata: Dict mapping model names to their metadata
+            - prices: Dict mapping model names to their prices
+            - no_cost_models: List of model names that have no associated cost
+        """
+        metadata = dict(MODEL_METADATA_MAPPING)
+        prices = dict(OUTPUT_PRICE_PER_MILLION_TOKEN)
+        no_cost = list(NO_COST_MODELS)
+
+        # Check for additional metadata config file path in environment variables
+        metadata_config_path = os.getenv("BFCL_MODEL_METADATA")
+
+        if metadata_config_path and os.path.exists(metadata_config_path):
+            try:
+                with open(metadata_config_path) as f:
+                    custom_config = json.load(f)
+
+                # Add custom model metadata
+                if "metadata" in custom_config:
+                    metadata.update(custom_config["metadata"])
+
+                # Add custom pricing information
+                if "prices" in custom_config:
+                    prices.update(custom_config["prices"])
+
+                # Add additional no-cost models
+                if "no_cost_models" in custom_config:
+                    no_cost.extend(custom_config["no_cost_models"])
+
+            except Exception as e:
+                print(f"Error loading custom metadata config: {str(e)}")
+
+        return metadata, prices, no_cost
+
+# Global metadata loader instance
+metadata_loader = MetadataLoader() 

berkeley-function-call-leaderboard/bfcl/model_handler/handler_loader.py

@@ -0,0 +1,61 @@
+import json
+import importlib.util
+import os
+from pathlib import Path
+from typing import Type, Optional
+
+from bfcl.model_handler.base_handler import BaseHandler
+from bfcl.model_handler.handler_map import HANDLER_MAP
+
+class HandlerLoader:
+    @staticmethod
+    def load_handler_class(module_path: str, class_name: str) -> Optional[Type[BaseHandler]]:
+        """Dynamically load handler classes from a specified path"""
+        try:
+            abs_path = str(Path(module_path).resolve())
+            spec = importlib.util.spec_from_file_location("custom_module", abs_path)
+            if spec is None or spec.loader is None:
+                raise ImportError(f"Could not load spec for module: {module_path}")
+
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+
+            handler_class = getattr(module, class_name, None)
+            if handler_class is None:
+                raise AttributeError(f"Class {class_name} not found in {module_path}")
+
+            # Checking for BaseHandler Inheritance
+            if not issubclass(handler_class, BaseHandler):
+                raise TypeError(f"Class {class_name} must inherit from BaseHandler")
+
+            return handler_class
+
+        except Exception as e:
+            print(f"Error loading handler class {class_name} from {module_path}: {str(e)}")
+            return None
+
+    @staticmethod
+    def get_handler_class(model_name: str) -> Optional[Type[BaseHandler]]:
+        """Returns the handler class corresponding to the model name"""
+        # Check the path to the handler mapping file in an environment variable
+        handler_config_path = os.getenv("BFCL_HANDLER_CONFIG")
+
+        if handler_config_path and os.path.exists(handler_config_path):
+            try:
+                with open(handler_config_path) as f:
+                    handler_config = json.load(f)
+
+                if model_name in handler_config:
+                    config = handler_config[model_name]
+                    handler_class = HandlerLoader.load_handler_class(
+                        config["module_path"], 
+                        config["class_name"]
+                    )
+                    if handler_class:
+                        return handler_class
+
+            except Exception as e:
+                print(f"Error loading custom handler config: {str(e)}")
+
+        # Lookup in the default handler map
+        return HANDLER_MAP.get(model_name)