diff --git a/fastdeploy/entrypoints/llm.py b/fastdeploy/entrypoints/llm.py
index 1a66c472f53..cd380761730 100644
--- a/fastdeploy/entrypoints/llm.py
+++ b/fastdeploy/entrypoints/llm.py
@@ -23,12 +23,14 @@
import uuid
from typing import Any, Optional, Union
+from pydantic import ValidationError
from tqdm import tqdm
from fastdeploy.engine.args_utils import EngineArgs
from fastdeploy.engine.engine import LLMEngine
from fastdeploy.engine.sampling_params import SamplingParams
from fastdeploy.entrypoints.chat_utils import load_chat_template
+from fastdeploy.entrypoints.openai.protocol import ChatCompletionToolsParam
from fastdeploy.entrypoints.openai.tool_parsers import ToolParserManager
from fastdeploy.utils import (
deprecated_kwargs_warning,
@@ -204,6 +206,7 @@ def chat(
use_tqdm: bool = True,
chat_template_kwargs: Optional[dict[str, Any]] = None,
chat_template: Optional[str] = None,
+ tools: Optional[Union[ChatCompletionToolsParam, list[ChatCompletionToolsParam]]] = None,
stream: bool = False,
):
"""
@@ -243,6 +246,12 @@ def chat(
if chat_template is None:
chat_template = self.chat_template
+ validated_tools = None
+ if tools is not None:
+ try:
+ validated_tools = self._validate_tools(tools)
+ except ValueError as e:
+ raise RuntimeError(f"Failed to validate 'tools' parameter in chat method: {e}") from e
messages_len = len(messages)
for i in range(messages_len):
messages[i] = {"messages": messages[i]}
@@ -251,6 +260,7 @@ def chat(
sampling_params=sampling_params,
chat_template_kwargs=chat_template_kwargs,
chat_template=chat_template,
+ tools=validated_tools,
)
topk_logprobs = sampling_params[0].logprobs if sampling_params_len > 1 else sampling_params.logprobs
@@ -310,6 +320,8 @@ def _add_request(
if current_sampling_params.guided_decoding is not None:
guided_decoding_dict = current_sampling_params.guided_decoding.to_dict()
tasks.update(guided_decoding_dict)
+ if kwargs.get("tools") is not None:
+ tasks["tools"] = kwargs.get("tools")
self.llm_engine.add_requests(tasks, current_sampling_params, **kwargs)
return req_ids
@@ -558,6 +570,60 @@ def _create_incremental_result(self, current_result, previous_count, pos, prompt
return incremental_result
+ def _validate_tools(self, raw_tools: Any) -> Optional[list[dict]]:
+ """
+ Validate the format of the `tools` parameter for chat requests.
+ Valid inputs are accepted and standardized, while invalid inputs raise ValueError.
+ Empty dict/list will be returned as None.
+
+ Args:
+ raw_tools: Raw `tools` parameter obtained from kwargs (can be any type)
+
+ Returns:
+ Optional[List[Dict[str, Any]]]: Standardized list of valid tool dictionaries if validation passes;
+ None if `raw_tools` is None or empty (empty dict/list).
+
+ Raises:
+ ValueError: Raised when input type is invalid or format does not meet standards.
+ """
+ if raw_tools is None:
+ return None
+ if isinstance(raw_tools, ChatCompletionToolsParam):
+ return [raw_tools]
+ if isinstance(raw_tools, list) and all(isinstance(t, ChatCompletionToolsParam) for t in raw_tools):
+ if not raw_tools:
+ return None
+ else:
+ return raw_tools
+
+ if not isinstance(raw_tools, dict) and not isinstance(raw_tools, list):
+ raise ValueError(
+ f"Invalid tools top-level type! Expected None, dict (single tool) or list (multiple tools), "
+ f"but got type '{type(raw_tools).__name__}' (value: {raw_tools})."
+ )
+ tools_list: list[dict[str, Any]] = [raw_tools] if isinstance(raw_tools, dict) else raw_tools
+
+ if not tools_list:
+ return None
+
+ validated_tools = []
+ for idx, tool in enumerate(tools_list):
+ if not isinstance(tool, dict):
+ raise ValueError(
+ f"Invalid element type in tools list! At index {idx}, "
+ f"expected dict (tool definition), but got type '{type(tool).__name__}' (value: {tool})."
+ )
+
+ try:
+ validated_tool_obj = ChatCompletionToolsParam.model_validate(tool)
+ validated_tools.append(validated_tool_obj.model_dump())
+ except ValidationError as e:
+ raise ValueError(
+ f"Invalid tool format at index {idx} in tools list! " f"Tool content: {tool}\nError details: {e}"
+ ) from e
+
+ return validated_tools
+
if __name__ == "__main__":
# llm = LLM(model="llama_model")
diff --git a/tests/entrypoints/test_chat.py b/tests/entrypoints/test_chat.py
index 0078cd8a18e..7167ce19aa0 100644
--- a/tests/entrypoints/test_chat.py
+++ b/tests/entrypoints/test_chat.py
@@ -19,6 +19,7 @@
import weakref
from fastdeploy.entrypoints.llm import LLM
+from fastdeploy.entrypoints.openai.protocol import ChatCompletionToolsParam
MODEL_NAME = os.getenv("MODEL_PATH") + "/ERNIE-4.5-0.3B-Paddle"
@@ -58,6 +59,133 @@ def test_chat(self):
outputs = self.llm.chat(messages=self.PROMPTS, sampling_params=None)
self.assertEqual(len(self.PROMPTS), len(outputs))
+ def test_chat_with_tools(self):
+ """Test chat with tools:
+ 1. spliced_message (after chat_template) contains tool-related content
+ 2. Model output contains tool_call
+ """
+ prompts = [{"role": "user", "content": "北京海淀区今天天气怎么样？用摄氏度表示温度。"}]
+ tools = [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Determine weather in my location",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {"type": "string", "description": "The city and state e.g. San Francisco, CA"},
+ "unit": {"type": "string", "enum": ["c", "f"]},
+ },
+ "additionalProperties": False,
+ "required": ["location", "unit"],
+ },
+ "strict": True,
+ },
+ }
+ ]
+ chat_template = "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in content %}\n {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- set content = content.split('')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content.strip('\\n') + '\\n\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '\\n\\n\\n\\n' }}\n {%- endif %}\n{%- endif %}"
+
+ data_processor = self.llm.llm_engine.data_processor
+ captured_spliced_message = None
+
+ def capture_spliced_message(request_or_messages, **kwargs):
+ """Wrap original messages2ids to capture spliced_message"""
+ token_ids = data_processor.original_messages2ids(request_or_messages, **kwargs)
+ nonlocal captured_spliced_message
+ captured_spliced_message = request_or_messages.get("prompt_tokens")
+ return token_ids
+
+ data_processor.original_messages2ids = data_processor.messages2ids
+ data_processor.messages2ids = capture_spliced_message
+
+ try:
+ outputs = self.llm.chat(
+ messages=prompts,
+ tools=tools,
+ chat_template=chat_template,
+ chat_template_kwargs={"enable_thinking": False},
+ stream=False,
+ )
+
+ self.assertIsNotNone(captured_spliced_message, "Failed to capture spliced_message from messages2ids")
+ self.assertIn(
+ "",
+ captured_spliced_message,
+ f"spliced_message '{captured_spliced_message}' missing tag (chat_template not applied)",
+ )
+
+ output = outputs[0]
+ self.assertEqual(len(prompts), len(outputs))
+ self.assertTrue(hasattr(output, "outputs"))
+ self.assertTrue(hasattr(output.outputs, "text"))
+ finally:
+ data_processor.messages2ids = data_processor.original_messages2ids
+
+ def test_validate_tools(self):
+ """Test both valid and invalid scenarios for _validate_tools method"""
+ # Prepare valid test data
+ valid_tool_dict = {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Get real-time weather of a city",
+ "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
+ },
+ }
+ valid_tool_model = ChatCompletionToolsParam(**valid_tool_dict)
+ valid_model_list = [valid_tool_model, valid_tool_model]
+ valid_dict_list = [valid_tool_dict, valid_tool_dict]
+
+ # Test valid scenarios
+ # 1. Input is None
+ self.assertIsNone(self.llm._validate_tools(None))
+
+ # 2. Input is single ChatCompletionToolsParam instance
+ result = self.llm._validate_tools(valid_tool_model)
+ self.assertEqual(len(result), 1)
+ self.assertIsInstance(result[0], ChatCompletionToolsParam)
+
+ # 3. Input is list of ChatCompletionToolsParam instances
+ self.assertEqual(self.llm._validate_tools(valid_model_list), valid_model_list)
+
+ # 4. Input is single valid dict
+ result = self.llm._validate_tools(valid_tool_dict)
+ self.assertEqual(len(result), 1)
+ self.assertIsInstance(result[0], dict)
+ self.assertEqual(result[0]["type"], "function")
+
+ # 5. Input is list of valid dicts
+ result = self.llm._validate_tools(valid_dict_list)
+ self.assertEqual(len(result), 2)
+ self.assertIsInstance(result[1], dict)
+
+ # 6. Input is empty list
+ self.assertIsNone(self.llm._validate_tools([]))
+
+ # Test invalid scenarios (should raise ValueError)
+ # 1. Input is string (invalid top-level type)
+ with self.assertRaises(ValueError):
+ self.llm._validate_tools("invalid_string")
+
+ # 2. Input list contains non-dict element
+ with self.assertRaises(ValueError):
+ self.llm._validate_tools([valid_tool_dict, 123])
+
+ # 3. Tool dict missing required field (function.name)
+ invalid_tool_missing_name = {"type": "function", "function": {"description": "Missing 'name' field"}}
+ with self.assertRaises(ValueError):
+ self.llm._validate_tools(invalid_tool_missing_name)
+
+ # 4. Tool dict with wrong 'type' value
+ invalid_tool_wrong_type = {"type": "invalid_type", "function": {"name": "test", "description": "Wrong type"}}
+ with self.assertRaises(ValueError):
+ self.llm._validate_tools(invalid_tool_wrong_type)
+
+ # 5. Input is boolean
+ with self.assertRaises(ValueError):
+ self.llm._validate_tools(True)
+
if __name__ == "__main__":
unittest.main()