diff --git a/berkeley-function-call-leaderboard/README.md b/berkeley-function-call-leaderboard/README.md index 4acbf926a..097c3a85f 100644 --- a/berkeley-function-call-leaderboard/README.md +++ b/berkeley-function-call-leaderboard/README.md @@ -213,6 +213,7 @@ Below is _a table of models we support_ to run our leaderboard evaluation agains |Qwen/Qwen2-{1.5B,7B}-Instruct 💻| Prompt| |Team-ACE/ToolACE-8B 💻| Function Calling| |openbmb/MiniCPM3-4B 💻| Function Calling| +|KishoreK/ActionGemma-9B| Function Calling| Here {MODEL} 💻 means the model needs to be hosted locally and called by vllm, {MODEL} means the models that are called API calls. For models with a trailing `-FC`, it means that the model supports function-calling feature. You can check out the table summarizing feature supports among different models [here](https://gorilla.cs.berkeley.edu/blogs/8_berkeley_function_calling_leaderboard.html#prompt). diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py b/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py index eae669af0..faed18357 100644 --- a/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py +++ b/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py @@ -23,6 +23,7 @@ from bfcl.model_handler.proprietary_model.nvidia import NvidiaHandler from bfcl.model_handler.proprietary_model.openai import OpenAIHandler from bfcl.model_handler.proprietary_model.yi import YiHandler +from bfcl.model_handler.oss_model.actionGemma_handler import ActionGemmaHandler # TODO: Add Deepseek V2, meta-llama/Llama-3.1-405B-Instruct @@ -79,6 +80,7 @@ "command-r-plus-optimized": CohereHandler, "snowflake/arctic": NvidiaHandler, "nvidia/nemotron-4-340b-instruct": NvidiaHandler, + "KishoreK/ActionGemma-9B":ActionGemmaHandler, # "yi-large-fc": YiHandler, # Their API is under maintenance, and will not be back online in the near future } diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/oss_model/actionGemma_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/oss_model/actionGemma_handler.py new file mode 100644 index 000000000..b20a742d4 --- /dev/null +++ b/berkeley-function-call-leaderboard/bfcl/model_handler/oss_model/actionGemma_handler.py @@ -0,0 +1,144 @@ +import json + +from bfcl.model_handler.oss_model.base_oss_handler import OSSHandler +from bfcl.model_handler.model_style import ModelStyle + +SYSTEM_PROMPT =""" +system +You are an AI assistant for function calling. +For politically sensitive questions, security and privacy issues, +and other non-computer science questions, you will refuse to answer\n +""".strip() + +TASK_INSTRUCTION = """ +You are an expert in composing functions. You are given a question and a set of possible functions. +Based on the question, you will need to make one or more function/tool calls to achieve the purpose. +If none of the function can be used, point it out and refuse to answer. +If the given question lacks the parameters required by the function, also point it out. +""".strip() + + +FORMAT_INSTRUCTION = """ +The output MUST strictly adhere to the following JSON format, and NO other text MUST be included. +The example format is as follows. Please make sure the parameter type is correct. If no function call is needed, please make tool_calls an empty list '[]' +``` +{ + "tool_calls": [ + {"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}}, + ... (more tool calls as required) + ] +} +``` +""" + +class ActionGemmaHandler(OSSHandler): + def __init__(self, model_name, temperature=0.001) -> None: + super().__init__(model_name, temperature) + self.model_style = ModelStyle.OSSMODEL + + def _format_prompt(query, functions, test_category): + def convert_to_xlam_tool(tools): + '''Convert the Gorilla function call format to xlam format''' + if isinstance(tools, dict): + xlam_tools = { + "name": tools["name"], + "description": tools["description"], + "parameters": tools["parameters"].get("properties", {}) + } + required = tools["parameters"].get("required", []) + for param in required: + xlam_tools["parameters"][param]["required"] = True + elif isinstance(tools, list): + xlam_tools = [] + for tool in tools: + xlam_tools.append(convert_to_xlam_tool(tool)) + else: + xlam_tools = tools + return xlam_tools + + tools = convert_to_xlam_tool(functions) + if isinstance(tools, dict): + tools = [tools] + + content = f"\n{TASK_INSTRUCTION}\n\n" +# content += f"{FORMAT_INSTRUCTION}\n\n\n" + content += "\n" + json.dumps(tools) + "\n\n\n" + + content += f"user\n{query}\n\n" + return SYSTEM_PROMPT + f"\n{content}\nassistant" + + def inference( + self, test_question, num_gpus, gpu_memory_utilization, format_prompt_func=_format_prompt + ): + print(f"[INFO] >> {num_gpus}, {gpu_memory_utilization}") + return super().inference( + test_question, num_gpus, gpu_memory_utilization, format_prompt_func + ) + + def decode_ast(self,result,language="Python"): + result_list = self.convert_to_dict(result) + return result_list + + @staticmethod + def xlam_json_to_python_tool_calls(tool_calls): + """ + Converts a list of function calls in xLAM JSON format to Python format. + + Parameters: + tool_calls (list): A list of dictionaries, where each dictionary represents a function call in xLAM JSON format. + + Returns: + python_format (list): A list of strings, where each string is a function call in Python format. + """ + if not isinstance(tool_calls, list): + tool_calls = [tool_calls] + + python_format = [] + for tool_call in tool_calls: + if isinstance(tool_call, dict): + name = tool_call.get('name', "") + arguments = tool_call.get('arguments', {}) + args_str = ', '.join([f"{key}={repr(value)}" for key, value in arguments.items()]) + python_format.append(f"{name}({args_str})") + else: + print(f"Invalid format: {tool_call}") + + return python_format + + def decode_execute(self,result): + try: + result_json = json.loads(result) + except: + return result + if isinstance(result_json, list): + tool_calls = result_json + elif isinstance(result_json, dict): + tool_calls = result_json.get('tool_calls', []) + else: + tool_calls = [] + function_call = self.xlam_json_to_python_tool_calls(tool_calls) + return function_call + + def convert_to_dict(self, input_str): + """ + Convert a JSON-formatted string into a dictionary of tool calls and their arguments. + + Parameters: + - input_str (str): A JSON-formatted string containing 'tool_calls' with 'name' and 'arguments'. + + Returns: + - list[dict]: A list of dictionaries with tool call names as keys and their arguments as values. + """ + try: + data = json.loads(input_str) + except json.JSONDecodeError: + return input_str + + tool_calls = data if isinstance(data, list) else data.get('tool_calls', []) + + result_list = [ + {tool_call.get('name', ''): tool_call.get('arguments', {})} + for tool_call in tool_calls if isinstance(tool_call, dict) + ] + + return result_list