From 16eda8c43a9d62f3dc90e8a0149855b3820049c8 Mon Sep 17 00:00:00 2001 From: "Ye (Charlotte) Qi" Date: Fri, 11 Apr 2025 15:26:17 -0700 Subject: [PATCH] [Frontend] Added chat templates for LLaMa4 pythonic tool calling (#16463) Signed-off-by: Ye (Charlotte) Qi Co-authored-by: Kai Wu --- docs/source/features/tool_calling.md | 2 + .../tool_chat_template_llama4_pythonic.jinja | 139 ++++++++++++++++++ tests/tool_use/conftest.py | 25 +++- tests/tool_use/utils.py | 16 ++ .../tool_parsers/pythonic_tool_parser.py | 2 +- 5 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 examples/tool_chat_template_llama4_pythonic.jinja diff --git a/docs/source/features/tool_calling.md b/docs/source/features/tool_calling.md index 17cee6da..8b8bbd28 100644 --- a/docs/source/features/tool_calling.md +++ b/docs/source/features/tool_calling.md @@ -245,6 +245,8 @@ Example supported models: * `meta-llama/Llama-3.2-3B-Instruct`\* (use with `examples/tool_chat_template_llama3.2_pythonic.jinja`) * `Team-ACE/ToolACE-8B` (use with `examples/tool_chat_template_toolace.jinja`) * `fixie-ai/ultravox-v0_4-ToolACE-8B` (use with `examples/tool_chat_template_toolace.jinja`) +* `meta-llama/Llama-4-Scout-17B-16E-Instruct`\* (use with `examples/tool_chat_template_llama4_pythonic.jinja`) +* `meta-llama/Llama-4-Maverick-17B-128E-Instruct`\* (use with `examples/tool_chat_template_llama4_pythonic.jinja`) Flags: `--tool-call-parser pythonic --chat-template {see_above}` diff --git a/examples/tool_chat_template_llama4_pythonic.jinja b/examples/tool_chat_template_llama4_pythonic.jinja new file mode 100644 index 00000000..bd18a35b --- /dev/null +++ b/examples/tool_chat_template_llama4_pythonic.jinja @@ -0,0 +1,139 @@ +{{- bos_token }} +{%- if custom_tools is defined %} + {%- set tools = custom_tools %} +{%- endif %} +{%- if not tools_in_user_message is defined %} + {%- set tools_in_user_message = false %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = none %} +{%- endif %} + +{#- This block extracts the system message, so we can slot it into the right place. #} +{%- if messages[0]['role'] == 'system' %} + {%- if messages[0]['content'] is string %} + {%- set system_message = messages[0]['content']|trim %} + {%- else %} + {%- set system_message = messages[0]['content'][0]['text']|trim %} + {%- endif %} + {%- set messages = messages[1:] %} +{%- else %} + {%- if tools is not none %} + {#- Add default tool system message when tools are provided #} + {%- set system_message = "You are a helpful assistant with tool calling " + "capabilities. Only reply with a tool call if the function exists in the " + "library provided by the user. If it doesn't exist, just reply directly in " + "natural language. When you receive a tool call response, use the output to " + "format an answer to the original user question." %} + {%- else %} + {%- set system_message = "" %} + {%- endif %} +{%- endif %} + +{#- System message if the user supplied one, or if tools are used (default tool system message) #} +{%- if system_message %} + {#- always use user provided system message to override default tool system message #} + {{- "<|header_start|>system<|header_end|>\n\n" }} + {{- system_message }} + {%- if tools is not none and not tools_in_user_message %} + {{- "Tools: You have access to the following tools. You might need to use one " + "or more function/tool calls to fulfill the task. \n" + "If none are needed, then proceed to the response.\n\n" + "Tool Call Syntax: You can call tools using the following syntax:\n" + "[func_name1(params_name1=params_value1, params_name2=params_value2, ...), ...]\n" + "Do not include anything else when calling the tools with the syntax above.\n\n" + "Here is a list of functions in JSON format that you can invoke.\n " }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} + {%- endif %} + {{- "<|eot|>" }} +{%- endif %} + +{#- Custom tools are passed in a user message with some extra guidance #} +{%- if tools_in_user_message and tools is not none %} + {#- Extract the first user message so we can plug it in here #} + {%- if messages | length != 0 %} + {%- if messages[0]['content'] is string %} + {%- set first_user_message = messages[0]['content']|trim %} + {%- else %} + {%- set first_user_message = messages[0]['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | map('trim') | join('\n') %} + {%- endif %} + {%- set messages = messages[1:] %} + {%- else %} + {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} + {%- endif %} + {{- '<|header_start|>user<|header_end|>\n\n' -}} + {{- first_user_message}} + {{- "\nHere is a list of functions in JSON format that you can invoke:"}} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} + {{- "Should you decide to return the function call(s), put them in the format " + "of [func_name1(params_name1=params_value1, params_name2=params_value2, " + "...), ...]\nDo not include anything else when calling the tools with the " + "syntax above." }} +{%- endif %} + +{%- for message in messages %} + {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} + {{- '<|header_start|>' + message['role'] + '<|header_end|>\n\n' }} + {%- if message['content'] is string %} + {{- message['content'] }} + {%- else %} + {%- for content in message['content'] %} + {%- if content['type'] == 'image' %} + {{- '<|image|>' }} + {%- elif content['type'] == 'text' %} + {{- content['text'] | trim }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- "<|eot|>" }} + {%- elif 'tool_calls' in message and message.tool_calls|length > 0 %} + {%- set tool_call = message.tool_calls[0].function %} + {{- '<|header_start|>assistant<|header_end|>\n\n' -}} + {%- if message['content'] is string %} + {{- message['content'] }} + {%- else %} + {%- for content in message['content'] %} + {%- if content['type'] == 'image' %} + {{- '<|image|>' }} + {%- elif content['type'] == 'text' %} + {{- content['text'] }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- tool_call.name + '(' -}} + {%- for param in tool_call.arguments %} + {{- param + '=' -}} + {{- "%s" | format(tool_call.arguments[param]) -}} + {% if not loop.last %}, {% endif %} + {%- endfor %} + {{- ')' -}} + {% if not loop.last %}, {% endif %} + {%- endfor %} + {{- "<|eom|>" }} + {%- elif message.role == "tool" or message.role == "ipython" %} + {{- "<|header_start|>ipython<|header_end|>\n\n" }} + {%- if message.content is string %} + {{- message.content | tojson }} + {%- else %} + {%- for content in message['content'] %} + {%- if content['type'] == 'text' %} + {{- content['text'] | tojson }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- "<|eom|>" }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|header_start|>assistant<|header_end|>\n\n' }} +{%- endif %} diff --git a/tests/tool_use/conftest.py b/tests/tool_use/conftest.py index 39ab01c9..4bf9b45f 100644 --- a/tests/tool_use/conftest.py +++ b/tests/tool_use/conftest.py @@ -10,10 +10,33 @@ from vllm.platforms import current_platform from .utils import ARGS, CONFIGS, ServerConfig +# select models to test based on command line arguments +def pytest_addoption(parser): + parser.addoption("--models", + nargs="+", + help="Specify one or more models to test") + parser.addoption("--extended", + action="store_true", + default=False, + help="invoke extended tests requiring large GPUs") + + # for each server config, download the model and return the config @pytest.fixture(scope="session", params=CONFIGS.keys()) def server_config(request): - config = CONFIGS[request.param] + extended = request.config.getoption("--extended") + models = request.config.getoption("--models") + + config_keys_to_test = [ + key for key in CONFIGS if (models is None or key in models) and ( + extended or not CONFIGS[key].get("extended", False)) + ] + + config_key = request.param + if config_key not in config_keys_to_test: + pytest.skip(f"Skipping config '{config_key}'") + + config = CONFIGS[config_key] if current_platform.is_rocm() and not config.get("supports_rocm", True): pytest.skip("The {} model can't be tested on the ROCm platform".format( diff --git a/tests/tool_use/utils.py b/tests/tool_use/utils.py index 231e4aad..7c87c73f 100644 --- a/tests/tool_use/utils.py +++ b/tests/tool_use/utils.py @@ -16,6 +16,7 @@ class ServerConfig(TypedDict, total=False): system_prompt: Optional[str] supports_parallel: Optional[bool] supports_rocm: Optional[bool] + extended: Optional[bool] # tests do not run in CI automatically def patch_system_prompt(messages: list[dict[str, Any]], @@ -82,6 +83,21 @@ CONFIGS: dict[str, ServerConfig] = { "supports_parallel": False, }, + "llama4": { + "model": + "meta-llama/Llama-4-Scout-17B-16E-Instruct", + "arguments": [ + "--enforce-eager", "--no-enable-prefix-caching", + "--tool-call-parser", "pythonic", "--chat-template", + str(VLLM_PATH / + "examples/tool_chat_template_llama4_pythonic.jinja"), "-tp", + "4" + ], + "supports_parallel": + False, + "extended": + True + }, "mistral": { "model": "mistralai/Mistral-7B-Instruct-v0.3", diff --git a/vllm/entrypoints/openai/tool_parsers/pythonic_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/pythonic_tool_parser.py index 1b9317f1..9f141d6b 100644 --- a/vllm/entrypoints/openai/tool_parsers/pythonic_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/pythonic_tool_parser.py @@ -28,7 +28,7 @@ class _UnexpectedAstError(Exception): class PythonicToolParser(ToolParser): """ Tool call parser for models that produce tool calls in a pythonic style, - such as Llama 3.2 models. + such as Llama 3.2 and Llama 4 models. Used when --enable-auto-tool-choice --tool-call-parser pythonic are all set """