From ee692375cf00fc4a76142b3c4dbd5e1d4721d15c Mon Sep 17 00:00:00 2001 From: Yueh-Ting Chen Date: Thu, 22 Oct 2026 14:30:22 +0800 Subject: [PATCH] [https://nvbugs/6932723][feat] Require Nemotron tool calls Enable required tool choice for native Nemotron serving, including Super 3.5 VL, with XGrammar and model-specific serving extensions. Resolve thinking state from the rendered prompt and relay it between context and generation workers. Preserve tool arguments across streaming boundaries and recover omitted parameter closers in unconstrained Chat Completions and Responses output. Keep required-grammar argument boundaries and partial previews intact. Cover grammar constraints, extension policies, request validation, reasoning-mode resolution, and incremental tool parsing. Signed-off-by: Yueh-Ting Chen --- tensorrt_llm/llmapi/reasoning_parser.py | 60 +- tensorrt_llm/serve/extensions/__init__.py | 4 +- tensorrt_llm/serve/extensions/nemotron.py | 29 + tensorrt_llm/serve/openai_server.py | 63 +- tensorrt_llm/serve/postprocess_handlers.py | 4 +- tensorrt_llm/serve/render/chat.py | 2 +- tensorrt_llm/serve/responses_utils.py | 15 +- tensorrt_llm/serve/serving_extensions.py | 18 +- .../serve/tool_parser/base_tool_parser.py | 13 + .../serve/tool_parser/qwen3_coder_parser.py | 324 ++++++--- .../serve/tool_parser/tool_parser_factory.py | 11 +- .../nemotron_super_vl_chat_template.jinja | 251 +++++++ .../apps/test_reasoning_prompt_resolution.py | 9 +- .../unittest/llmapi/apps/test_render_core.py | 12 +- .../test_responses_streaming_tool_calls.py | 158 ++++- .../llmapi/apps/test_serving_extensions.py | 300 ++++++++- .../unittest/llmapi/apps/test_tool_parsers.py | 625 ++++++++++++++++++ .../unittest/llmapi/test_reasoning_parser.py | 63 +- 18 files changed, 1820 insertions(+), 141 deletions(-) create mode 100644 tensorrt_llm/serve/extensions/nemotron.py create mode 100644 tests/unittest/llmapi/apps/data/nemotron_super_vl_chat_template.jinja diff --git a/tensorrt_llm/llmapi/reasoning_parser.py b/tensorrt_llm/llmapi/reasoning_parser.py index 9863181705da..127039547764 100644 --- a/tensorrt_llm/llmapi/reasoning_parser.py +++ b/tensorrt_llm/llmapi/reasoning_parser.py @@ -768,6 +768,10 @@ class NemotronV3ReasoningParser(DeepSeekR1Parser): The `enable_thinking` flag is read from `chat_template_kwargs`. """ + resolves_thinking_from_prompt = True + reasoning_start = "" + reasoning_end = "" + def __init__(self, *, reasoning_at_start: bool = True, @@ -803,32 +807,35 @@ def _maybe_swap_content( return result def parse_delta(self, delta_text: str) -> ReasoningParserResult: - """Wraps the parent parse_delta to also treat `` as an - implicit end-of-reasoning marker. When the model omits `` - before generating a tool call, the tag would otherwise be absorbed - into reasoning_content and the downstream tool parser would never - see it (NVBug 6082303). - - `` is a special token that always arrives as a single - atomic delta, so we only need to check `delta_text` (not the - parent's internal buffer).""" - if (self.in_reasoning and self._tool_call_start in delta_text - and self.reasoning_end not in self._buffer): - remaining = self._buffer + """End reasoning at the first closing marker or tool-call opener.""" + buffered = self._buffer + delta_text + tool_idx = buffered.find(self._tool_call_start) + end_idx = buffered.find(self.reasoning_end) + if self.in_reasoning and tool_idx != -1 and (end_idx == -1 + or tool_idx < end_idx): self._buffer = "" self.in_reasoning = False - # Guaranteed non-negative: guarded by `in delta_text` above. - tool_idx = delta_text.find(self._tool_call_start) - reasoning = remaining + delta_text[:tool_idx] - content = delta_text[tool_idx:] + reasoning = buffered[:tool_idx] + content = buffered[tool_idx:] if self._force_nonempty_content: self._found_closing_tag = True self._accumulated_reasoning = "" return ReasoningParserResult(content=content, reasoning_content=reasoning) + pending = "" + if self.in_reasoning and end_idx == -1 and tool_idx == -1: + for size in range( + min(len(buffered), + len(self._tool_call_start) - 1), 0, -1): + if buffered.endswith(self._tool_call_start[:size]): + pending = buffered[-size:] + self._buffer = "" + delta_text = buffered[:-size] + break was_in_reasoning = self.in_reasoning result = super().parse_delta(delta_text) + self._buffer += pending if self._force_nonempty_content: if result.reasoning_content: self._accumulated_reasoning += result.reasoning_content @@ -868,14 +875,19 @@ def finish(self) -> ReasoningParserResult: return ReasoningParserResult() def parse(self, text: str) -> ReasoningParserResult: - result = super().parse(text) - tc = (result.reasoning_content.find(self._tool_call_start) - if result.reasoning_content else -1) - if tc != -1: - result = ReasoningParserResult( - content=result.reasoning_content[tc:] + result.content, - reasoning_content=result.reasoning_content[:tc]) - return self._maybe_swap_content(result) + start = 0 if self.reasoning_at_start else text.find( + self.reasoning_start) + if start != -1: + if not self.reasoning_at_start: + start += len(self.reasoning_start) + tool_idx = text.find(self._tool_call_start, start) + end_idx = text.find(self.reasoning_end, start) + if tool_idx != -1 and (end_idx == -1 or tool_idx < end_idx): + return self._maybe_swap_content( + ReasoningParserResult( + content=text[tool_idx:], + reasoning_content=text[start:tool_idx])) + return self._maybe_swap_content(super().parse(text)) @register_reasoning_parser("gemma4") diff --git a/tensorrt_llm/serve/extensions/__init__.py b/tensorrt_llm/serve/extensions/__init__.py index 32100da9b10a..fcfb9988d2b4 100644 --- a/tensorrt_llm/serve/extensions/__init__.py +++ b/tensorrt_llm/serve/extensions/__init__.py @@ -25,6 +25,6 @@ turn imports the registry module. Nothing here may import ``openai_server``. """ -from . import gpt_oss, kimi_k3 +from . import gpt_oss, kimi_k3, nemotron -__all__ = ["gpt_oss", "kimi_k3"] +__all__ = ["gpt_oss", "kimi_k3", "nemotron"] diff --git a/tensorrt_llm/serve/extensions/nemotron.py b/tensorrt_llm/serve/extensions/nemotron.py new file mode 100644 index 000000000000..b4e2e7207cc3 --- /dev/null +++ b/tensorrt_llm/serve/extensions/nemotron.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Serving extension for Nemotron checkpoints using required-tool grammars.""" + +from tensorrt_llm.serve.serving_extensions import ( + RequiredToolChoiceMode, + ServingExtension, + register_serving_extension, +) + + +@register_serving_extension(model_types=("nemotron_h", "nemotron_h_omni")) +class NemotronServingExtension(ServingExtension): + """Use mandatory tool-call grammars for native Nemotron chat requests.""" + + def required_tool_choice_mode(self) -> RequiredToolChoiceMode: + return "grammar" diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 55a0d24ca776..f2211b808475 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -228,6 +228,34 @@ def _warn_unresolvable_thinking_once(reasoning_parser: str) -> None: "build that relays 'resolved_thinking'.") +def _validate_required_tool_grammar_request( + request: ChatCompletionRequest, sampling_params: SamplingParams, *, + guided_decoding_backend: Optional[str], tool_parser: Optional[str], + reasoning_parser: Optional[str]) -> None: + """Validate inputs for mandatory tool-call grammar construction.""" + if guided_decoding_backend != "xgrammar": + raise ValueError("Required tool grammars need the xgrammar backend.") + if not tool_parser: + raise ValueError("Required tool grammars need a tool parser.") + if not reasoning_parser or not ReasoningParserFactory.resolves_thinking_from_prompt( + reasoning_parser): + raise ValueError( + "Required tool grammars need a reasoning parser that resolves its " + "mode from the rendered prompt. Configure the server with " + "--reasoning_parser nemotron-v3 (or a compatible parser alias).") + if not request.tools: + raise ValueError("tool_choice='required' requires request-level tools.") + if (request.prompt_token_ids is not None or request.prompt_token_ids_b64 + or not request.add_generation_prompt + or request.disaggregated_params is not None): + raise ValueError("Required tool grammars need chat messages with " + "add_generation_prompt=True on a native server.") + if sampling_params.guided_decoding is not None: + raise ValueError( + "tool_choice='required' cannot be combined with response_format " + "or other guided-decoding settings.") + + def _configure_parser_special_token_decoding( sampling_params: SamplingParams, reasoning_parser_name: Optional[str], tool_parser_name: Optional[str], has_tools: bool) -> None: @@ -1987,6 +2015,9 @@ async def chat_stream_generator( extension = render_resources.extension tool_dicts = prepare_chat_request(request, render_resources).tool_dicts + required_mode = extension.required_tool_choice_mode() + required_tool_grammar = (request.tool_choice == "required" + and required_mode == "grammar") conversation: List[ConversationMessage] = [] # Pass the model vocabulary size so ``logit_bias`` can be # expanded into an embedding bias tensor in the sampler. @@ -1999,6 +2030,14 @@ async def chat_stream_generator( gather_generation_logits, reasoning_parser=self.generator.args.reasoning_parser, backend=self.generator.args.backend) + if required_tool_grammar: + _validate_required_tool_grammar_request( + request, + sampling_params, + guided_decoding_backend=self.generator.args. + guided_decoding_backend, + tool_parser=self.tool_parser, + reasoning_parser=self.generator.args.reasoning_parser) # Pre-render, so the mode may be unresolved. Safe because the # boundary lookup reads the parser class attributes, not the # branch `__init__` picked. @@ -2116,7 +2155,7 @@ async def chat_stream_generator( # via structural tags (only if response_format doesn't already # set guided decoding). Dynamic tools must be in the grammar # too, or their calls would be masked out. - if sampling_params.guided_decoding is None: + if sampling_params.guided_decoding is None and not required_tool_grammar: if (forced_tool_name is not None and _parser_extracts_forced_tool_calls( self.tool_parser)): @@ -2273,6 +2312,7 @@ async def chat_stream_generator( if request.mm_processor_kwargs: prompt["mm_processor_kwargs"] = request.mm_processor_kwargs + thinking = None postproc_args.reasoning_parser = self.generator.args.reasoning_parser # Templates that prefill / leave the marker in the # prompt, so the request kwargs alone cannot tell the parser which @@ -2280,7 +2320,6 @@ async def chat_stream_generator( if postproc_args.reasoning_parser and ( ReasoningParserFactory.resolves_thinking_from_prompt( postproc_args.reasoning_parser)): - thinking = None if rendered_prompt and request.add_generation_prompt: thinking = ReasoningParserFactory.resolve_prefilled_thinking( postproc_args.reasoning_parser, rendered_prompt) @@ -2299,6 +2338,26 @@ async def chat_stream_generator( "thinking": thinking, "enable_thinking": thinking, } + if required_tool_grammar: + if thinking is None: + raise ValueError( + "Cannot resolve required-tool reasoning state from the rendered prompt." + ) + parser = ToolParserFactory.create_tool_parser(self.tool_parser) + build_required = functools.partial( + parser.build_required_structural_tag, + request.tools, + thinking=thinking) + required_format = await asyncio.get_running_loop( + ).run_in_executor(self._input_proc_executor, build_required) + if required_format is None: + raise ValueError( + "The configured tool parser does not support required tools." + ) + sampling_params.guided_decoding = GuidedDecodingParams( + structural_tag=required_format.model_dump_json( + by_alias=True)) + postproc_args.tool_parser_grammar_constrained = True postproc_args.tool_parser = self.tool_parser postproc_args.tool_call_id_type = self.tool_call_id_type if conversation and conversation[-1].get( diff --git a/tensorrt_llm/serve/postprocess_handlers.py b/tensorrt_llm/serve/postprocess_handlers.py index 12080002dc2d..b1f496600db9 100644 --- a/tensorrt_llm/serve/postprocess_handlers.py +++ b/tensorrt_llm/serve/postprocess_handlers.py @@ -103,6 +103,7 @@ class ChatPostprocArgs(PostprocArgs): last_message_content: Optional[str] = None reasoning_parser: Optional[str] = None tool_parser: Optional[str] = None + tool_parser_grammar_constrained: bool = False reasoning_parser_dict: dict[int, BaseReasoningParser] = field( default_factory=dict) tool_parser_dict: dict[int, BaseToolParser] = field(default_factory=dict) @@ -299,7 +300,8 @@ def apply_tool_parser(args: ChatPostprocArgs, if output_index not in args.tool_parser_dict: args.tool_parser_dict[ output_index] = ToolParserFactory.create_tool_parser( - args.tool_parser) + args.tool_parser, + grammar_constrained=args.tool_parser_grammar_constrained) tool_parser = args.tool_parser_dict[output_index] if tool_parser is not None and tools is not None: diff --git a/tensorrt_llm/serve/render/chat.py b/tensorrt_llm/serve/render/chat.py index d1e27e0bc66b..b3988516aa98 100644 --- a/tensorrt_llm/serve/render/chat.py +++ b/tensorrt_llm/serve/render/chat.py @@ -92,7 +92,7 @@ def prepare_chat_request(request: "ChatCompletionRequest", res: RenderResources) ensure_request_chat_template_allowed(request, res.allow_request_chat_template) extension = res.extension extension.apply_chat_extensions(request) - if request.tool_choice == "required" and not extension.allows_required_tool_choice(): + if request.tool_choice == "required" and extension.required_tool_choice_mode() is None: # Schema-accepting "required" everywhere but enforcing it only for models # whose extension allows it would silently degrade to "auto" elsewhere; # reject loudly for models that cannot honor it. Still a ValueError for the diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index c4b632d0fed2..0a4086d3c8dd 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio @@ -1445,6 +1446,8 @@ def _apply_tool_parser( text: str, streaming: bool, tool_parser_dict: Optional[dict[int, BaseToolParser]] = None, + finished: bool = False, + partial: bool = False, ) -> Tuple[str, list[ToolCallItem]]: tool_parser: Optional[BaseToolParser] = None if tool_parser_id is not None and tools is not None: @@ -1460,10 +1463,15 @@ def _apply_tool_parser( if tool_parser is not None and tools is not None: if not streaming: - result = tool_parser.detect_and_parse(text, tools) + result = (tool_parser.parse_partial(text, tools) + if partial else tool_parser.detect_and_parse(text, tools)) else: result = tool_parser.parse_streaming_increment(text, tools) normal_text, calls = result.normal_text, result.calls + if streaming and finished: + tail = tool_parser.finish(tools) + normal_text += tail.normal_text + calls += tail.calls if not streaming: warn_if_tool_call_unparsed(tool_parser_id, tool_parser, text, calls) else: @@ -2159,6 +2167,7 @@ def _should_send_done_events( text=full_text, streaming=False, tool_parser_dict=tool_parser_dict, + partial=not finished_generation, ) # Detect reasoning -> text transition @@ -2278,8 +2287,7 @@ def check_parser(parser_id: Optional[str], chat_template_kwargs=reasoning_chat_template_kwargs(request), ) - if delta_text: - # TODO(JunyiXu-nv): handle tool calls in streaming mode + if delta_text or finished_generation: delta_text, calls = _apply_tool_parser( tool_parser_id=tool_parser_id, tools=available_tools, @@ -2287,6 +2295,7 @@ def check_parser(parser_id: Optional[str], text=delta_text, streaming=True, tool_parser_dict=tool_parser_dict, + finished=finished_generation, ) _responses_debug_log( diff --git a/tensorrt_llm/serve/serving_extensions.py b/tensorrt_llm/serve/serving_extensions.py index f801f18ca0a3..ef6a129e73db 100644 --- a/tensorrt_llm/serve/serving_extensions.py +++ b/tensorrt_llm/serve/serving_extensions.py @@ -25,7 +25,7 @@ (:func:`structured_output_format_for`). Built-in extensions live in :mod:`tensorrt_llm.serve.extensions`; importing -that package registers them. Both lookup functions import it on first use so +that package registers them. The hook lookup functions import it on first use so the registry is populated wherever it is consulted (``openai_protocol`` reaches it from ``to_sampling_params`` without going through ``openai_server``). The import is deferred to call time because the built-ins import @@ -34,12 +34,15 @@ import enum import importlib -from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Literal, Optional if TYPE_CHECKING: from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest +RequiredToolChoiceMode = Literal["template", "grammar"] + + class OutputMode(enum.Enum): """How a model's generated output is turned into an API response.""" @@ -103,6 +106,10 @@ def apply_chat_extensions(self, request) -> None: ``model_type`` this extension is registered for. """ + def required_tool_choice_mode(self) -> Optional[RequiredToolChoiceMode]: + """Return the required-tool policy; boolean permission selects template enforcement.""" + return "template" if self.allows_required_tool_choice() else None + def structured_output_format( self, content: Dict[str, Any], chat_template_kwargs: Optional[Dict[str, Any]] ) -> Optional[Dict[str, Any]]: @@ -189,3 +196,10 @@ def structured_output_format_for( _load_builtin_extensions() extension = _BY_REASONING_PARSER.get(reasoning_parser) if reasoning_parser else None return None if extension is None else extension.structured_output_format + + +def required_tool_choice_mode(model_type: Optional[str]) -> Optional[RequiredToolChoiceMode]: + """Return required-tool handling for native chat, or None if unsupported.""" + _load_builtin_extensions() + extension = _BY_MODEL_TYPE.get(model_type) if model_type else None + return None if extension is None else extension.required_tool_choice_mode() diff --git a/tensorrt_llm/serve/tool_parser/base_tool_parser.py b/tensorrt_llm/serve/tool_parser/base_tool_parser.py index 6756641e7e70..daf4f0147cf6 100644 --- a/tensorrt_llm/serve/tool_parser/base_tool_parser.py +++ b/tensorrt_llm/serve/tool_parser/base_tool_parser.py @@ -8,6 +8,7 @@ from partial_json_parser.core.exceptions import MalformedJSON from partial_json_parser.core.options import Allow +from xgrammar.structural_tag import StructuralTag from tensorrt_llm.logger import logger @@ -78,6 +79,7 @@ class BaseToolParser(ABC): streaming_matches_whole_parse: bool = False def __init__(self): + self.grammar_constrained: bool = False # Streaming state management # Buffer for accumulating incomplete patterns that arrive across multiple streaming chunks self._buffer = "" @@ -205,6 +207,11 @@ def _ends_with_partial_token(self, buffer: str, bot_token: str) -> int: return i return 0 + def parse_partial(self, text: str, + tools: List[Tool]) -> StreamingParseResult: + """Inspect unfinished output without updating the streaming state.""" + return self.detect_and_parse(text, tools) + def finish(self, tools: List[Tool]) -> StreamingParseResult: """Finalize a stream that ended before the format's closing markers. @@ -430,6 +437,12 @@ def build_strict_structural_tag_format( """ return None + def build_required_structural_tag( + self, tools: List[Tool], *, + thinking: bool) -> Optional[StructuralTag]: + """Build a mandatory native tool-call format, or return None if unsupported.""" + return None + @abstractmethod def structure_info(self) -> _GetInfoFunc: """ diff --git a/tensorrt_llm/serve/tool_parser/qwen3_coder_parser.py b/tensorrt_llm/serve/tool_parser/qwen3_coder_parser.py index 9fe897828753..df58be812790 100644 --- a/tensorrt_llm/serve/tool_parser/qwen3_coder_parser.py +++ b/tensorrt_llm/serve/tool_parser/qwen3_coder_parser.py @@ -21,6 +21,10 @@ import re from typing import Any, Dict, List, Tuple +import jsonschema +import xgrammar +from xgrammar import structural_tag + from tensorrt_llm.logger import logger from tensorrt_llm.serve.openai_protocol import ChatCompletionToolsParam as Tool from tensorrt_llm.serve.tool_parser.base_tool_parser import BaseToolParser @@ -57,9 +61,6 @@ def __init__(self): self.tool_call_regex = re.compile( r"(.*?)|(.*?)$", re.DOTALL ) - self.tool_call_function_regex = re.compile( - r"|| bool: return self.tool_call_start_token in text + def build_required_structural_tag( + self, tools: list[Tool], *, thinking: bool + ) -> structural_tag.StructuralTag: + """Build a mandatory grammar for Qwen-Coder XML tool calls. + + The serving layer must validate model eligibility before calling this; + selecting this parser alone does not enable required tool choice. + ``thinking`` must be resolved from the rendered prompt and indicates + whether generation starts inside an already-open ```` block. + """ + names = [tool.function.name for tool in tools] + if not names or len(set(names)) != len(names): + raise ValueError("Required tool calls need nonempty tools with unique names.") + if any(not re.fullmatch(r"[A-Za-z0-9_.-]+", name) for name in names): + raise ValueError("Tool names must contain only letters, digits, '_', '-', or '.'.") + tool_dicts = [tool.model_dump(exclude_none=True) for tool in tools] + for tool in tool_dicts: + schema = tool["function"].get("parameters") or {"type": "object"} + try: + jsonschema.validators.validator_for(schema).check_schema(schema) + except jsonschema.SchemaError as exc: + raise ValueError("Invalid required-tool parameter schema.") from exc + if schema.get("type") != "object": + raise ValueError("Required tools need object parameter schemas.") + if schema.get("additionalProperties", False) is not False: + raise ValueError("Required XML tools do not support additionalProperties.") + tool["function"]["parameters"] = {"properties": {}, **schema} + try: + template = xgrammar.get_builtin_structural_tag( + model="qwen_coder", tools=tool_dicts, reasoning=False + ) + for tag, tool in zip(template.format.tags, tool_dicts): + schema = tool["function"]["parameters"] + if not schema["properties"] and schema.keys() <= { + "type", + "properties", + "additionalProperties", + "required", + "minProperties", + "maxProperties", + "title", + "description", + "$schema", + }: + # Empty argument lists have no XML parameter blocks. + jsonschema.validate({}, schema) + tag.content = structural_tag.ConstStringFormat(value="") + else: + parameters = str( + xgrammar.Grammar.from_structural_tag( + structural_tag.StructuralTag(format=tag.content) + ) + ) + # Allow spacing after XML parameter terminals while preserving + # the excluded delimiter in string-value dispatch rules. + parameters = "\n".join( + re.sub( + r'"(?:\\.|[^"\\])*"', + lambda match: match[0] + r" [ \t\r\n]*" + if match[0] == '""' + else match[0], + line, + ) + if "::=" in line + else line + for line in parameters.splitlines() + ) + tag.content = structural_tag.GrammarFormat(grammar=parameters) + tag.end = "\n" + calls = template.format + calls.at_least_one = True + calls.triggers = [self.tool_call_start_token] + elements = [structural_tag.RegexFormat(pattern=r"[ \t\r\n]*"), calls] + if thinking: + # The opening is already in the prompt. + elements.insert( + 0, + structural_tag.TagFormat( + begin="", + content=structural_tag.AnyTextFormat(excludes=[self.tool_call_start_token]), + end="", + ), + ) + template.format = structural_tag.SequenceFormat(elements=elements) + xgrammar.Grammar.from_structural_tag(template) + except (RuntimeError, jsonschema.ValidationError) as exc: + raise ValueError("Invalid or unsupported required-tool parameter schema.") from exc + return template + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: normal, calls = self._extract(text, tools) return StreamingParseResult(normal_text=normal, calls=calls) + def parse_partial(self, text: str, tools: List[Tool]) -> StreamingParseResult: + normal, calls = self._extract(text, tools, complete=False) + return StreamingParseResult(normal_text=normal, calls=calls) + def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> StreamingParseResult: + return self._parse_streaming(new_text, tools, complete=False) + + def _parse_streaming( + self, new_text: str, tools: List[Tool], *, complete: bool + ) -> StreamingParseResult: self._buf += new_text normal = "" calls: List[ToolCallItem] = [] @@ -93,8 +192,13 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami while True: # If we're not in a tool call and don't see a start token, return normal text if not self._in_tool_call and self.tool_call_start_token not in self._buf: - normal += self._buf - self._buf = "" + partial = self._ends_with_partial_token(self._buf, self.tool_call_start_token) + if partial: + normal += self._buf[:-partial] + self._buf = self._buf[-partial:] + else: + normal += self._buf + self._buf = "" break # Look for tool call start @@ -121,43 +225,54 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami # We're in a tool call, try to parse function name if not sent yet if not self._function_name_sent: # Look for function name pattern: - function_match = re.search(r"]+)>", self._buf) + function_match = re.search(r"]*)>", self._buf) if function_match: function_name = function_match.group(1).strip() - # Validate function name - if function_name: - self._current_function_name = function_name - self._function_name_sent = True - - # Initialize tool call tracking - if self.current_tool_id == -1: - self.current_tool_id = 0 - - # Ensure tracking arrays are large enough - while len(self.prev_tool_call_arr) <= self.current_tool_id: - self.prev_tool_call_arr.append({}) - while len(self.streamed_args_for_tool) <= self.current_tool_id: - self.streamed_args_for_tool.append("") - - # Store tool call info - self.prev_tool_call_arr[self.current_tool_id] = { - "name": function_name, - "arguments": {}, - } - - # Send tool name with empty parameters - calls.append( - ToolCallItem( - tool_index=self.current_tool_id, - name=function_name, - parameters="", - ) + if not function_name: + end_pos = self._find_closing_tag( + self._buf, + self.tool_call_end_token, + function_match.end(), + complete=complete, ) - - # Remove the processed function declaration - self._buf = self._buf[function_match.end() :] + if end_pos == -1: + # Keep the call buffered until all its markup can be discarded. + break + self._buf = self._buf[end_pos + len(self.tool_call_end_token) :] + self._reset_streaming_state() continue + self._current_function_name = function_name + self._function_name_sent = True + + # Initialize tool call tracking + if self.current_tool_id == -1: + self.current_tool_id = 0 + + # Ensure tracking arrays are large enough + while len(self.prev_tool_call_arr) <= self.current_tool_id: + self.prev_tool_call_arr.append({}) + while len(self.streamed_args_for_tool) <= self.current_tool_id: + self.streamed_args_for_tool.append("") + + # Store tool call info + self.prev_tool_call_arr[self.current_tool_id] = { + "name": function_name, + "arguments": {}, + } + + # Send tool name with empty parameters + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=function_name, + parameters="", + ) + ) + + # Remove the processed function declaration + self._buf = self._buf[function_match.end() :] + continue else: # Function name not complete yet, wait for more text break @@ -166,30 +281,33 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami if self._function_name_sent: # Process parameters and get any calls to emit param_config = self._get_arguments_config(self._current_function_name, tools) - parameter_calls = self._parse_and_stream_parameters(self._buf, param_config) + # A single increment can contain several calls. Parse only this call's body. + end_pos = self._find_closing_tag( + self._buf, self.tool_call_end_token, complete=complete + ) + body = self._buf if end_pos == -1 else self._buf[:end_pos] + if end_pos != -1: + function_end = self._find_closing_tag( + self._buf[: end_pos + len(self.tool_call_end_token)], + "", + complete=True, + ) + if function_end != -1: + body = self._buf[:function_end] + parameter_calls = self._parse_and_stream_parameters( + body, param_config, complete=end_pos != -1 + ) calls.extend(parameter_calls) # Check if tool call is complete - if self.tool_call_end_token in self._buf: - end_pos = self._buf.find(self.tool_call_end_token) - - # Add closing brace to complete the JSON object + if end_pos != -1: + # Parameter deltas leave the outer object open until the call closes. current_streamed = self.streamed_args_for_tool[self.current_tool_id] - if current_streamed: - # Count opening and closing braces to check if JSON is complete - open_braces = current_streamed.count("{") - close_braces = current_streamed.count("}") - if open_braces > close_braces: - calls.append( - ToolCallItem( - tool_index=self.current_tool_id, - name=None, - parameters="}", - ) - ) - self.streamed_args_for_tool[self.current_tool_id] = ( - current_streamed + "}" - ) + closing = "}" if current_streamed else "{}" + calls.append( + ToolCallItem(tool_index=self.current_tool_id, name=None, parameters=closing) + ) + self.streamed_args_for_tool[self.current_tool_id] = current_streamed + closing # Complete the tool call self._buf = self._buf[end_pos + len(self.tool_call_end_token) :] @@ -202,33 +320,31 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami return StreamingParseResult(normal_text=normal, calls=calls) + def finish(self, tools: List[Tool]) -> StreamingParseResult: + result = self._parse_streaming("", tools, complete=True) + if not self._in_tool_call: + result.normal_text += self._buf + self._buf = "" + self._reset_streaming_state() + return result + def _parse_and_stream_parameters( - self, text_to_parse: str, param_config: dict + self, text_to_parse: str, param_config: dict, *, complete: bool = False ) -> List[ToolCallItem]: - """Parse complete parameter blocks from text and return any tool call items to emit. - - This method: - 1. Finds all complete blocks - 2. Parses them into a dictionary - 3. Compares with current parameters and generates diff if needed - 4. Updates internal state - - Args: - text_to_parse: The text to search for parameter blocks + """Emit argument deltas for the current function body. - Returns: - List of ToolCallItem objects to emit (may be empty) + Once the body is complete, unconstrained output can recover its final + parameter without a closing marker. """ calls: List[ToolCallItem] = [] - # Find all complete parameter patterns - param_matches = list( - re.finditer(r"]+)>(.*?)", text_to_parse, re.DOTALL) - ) - - # Build new parameters dictionary new_params = {} - for match in param_matches: + pattern = ( + r"]+)>(.*?)(?:|$)" + if (complete and not self.grammar_constrained) + else r"]+)>(.*?)" + ) + for match in re.finditer(pattern, text_to_parse, re.DOTALL): param_name = match.group(1).strip() param_value = match.group(2) new_params[param_name] = self._convert_param_value( @@ -298,7 +414,41 @@ def _reset_streaming_state(self): self._streamed_parameters = {} self.current_tool_name_sent = False - def _extract(self, text: str, tools: List[Tool]) -> Tuple[str, List[ToolCallItem]]: + def _find_closing_tag( + self, text: str, end_token: str, start: int = 0, *, complete: bool = False + ) -> int: + """Find an outer closer, recovering unclosed parameters in unconstrained output. + + A complete next-call header takes precedence over an unclosed parameter + in unconstrained output. Constrained output keeps that markup as literal + argument text. A final call can recover only when no parameter closer remains. + """ + cursor = start + while True: + end = text.find(end_token, cursor) + parameter = text.find("", parameter) + if not self.grammar_constrained: + limit = parameter_end if parameter_end != -1 else len(text) + for boundary in re.finditer(r"\s*", text[parameter:limit]): + boundary_start = parameter + boundary.start() + boundary_end = parameter + boundary.end() + next_call = re.match( + r"\s*\s*]+)>", text[boundary_end:limit] + ) + if (next_call and next_call.group(1).strip()) or ( + complete and parameter_end == -1 + ): + return text.find(end_token, boundary_start, boundary_end) + if parameter_end == -1: + return -1 + cursor = parameter_end + len("") + + def _extract( + self, text: str, tools: List[Tool], *, complete: bool = True + ) -> Tuple[str, List[ToolCallItem]]: normal_parts: List[str] = [] calls: List[ToolCallItem] = [] cursor = 0 @@ -308,7 +458,7 @@ def _extract(self, text: str, tools: List[Tool]) -> Tuple[str, List[ToolCallItem normal_parts.append(text[cursor:]) break normal_parts.append(text[cursor:s]) - e = text.find(self.tool_call_end_token, s) + e = self._find_closing_tag(text, self.tool_call_end_token, s, complete=complete) if e == -1: normal_parts.append(text[s:]) break @@ -319,13 +469,17 @@ def _extract(self, text: str, tools: List[Tool]) -> Tuple[str, List[ToolCallItem def _parse_block(self, block: str, tools: List[Tool]) -> List[ToolCallItem]: res: List[ToolCallItem] = [] - for m in self.tool_call_function_regex.findall(block): - txt = m[0] if m[0] else m[1] - if ">" not in txt: + cursor = 0 + while match := re.search(r"]*)>", block[cursor:]): + fname = match.group(1).strip() + body_start = cursor + match.end() + body_end = self._find_closing_tag(block, "", body_start, complete=True) + if body_end == -1: + body_end = len(block) + body = block[body_start:body_end] + cursor = body_end + len("") + if not fname: continue - idx = txt.index(">") - fname = txt[:idx].strip() - body = txt[idx + 1 :] params: Dict[str, Any] = {} for pm in self.tool_call_parameter_regex.findall(body): ptxt = pm[0] if pm[0] else pm[1] diff --git a/tensorrt_llm/serve/tool_parser/tool_parser_factory.py b/tensorrt_llm/serve/tool_parser/tool_parser_factory.py index 97f9aa8abbeb..877a630e17fb 100644 --- a/tensorrt_llm/serve/tool_parser/tool_parser_factory.py +++ b/tensorrt_llm/serve/tool_parser/tool_parser_factory.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import json from pathlib import Path from typing import Optional, Type @@ -82,12 +85,16 @@ class ToolParserFactory: } @staticmethod - def create_tool_parser(tool_parser: str) -> BaseToolParser: + def create_tool_parser( + tool_parser: str, *, grammar_constrained: bool = False + ) -> BaseToolParser: try: tool_parser_class = ToolParserFactory.parsers[tool_parser.lower()] - return tool_parser_class() except KeyError as e: raise ValueError( f"Invalid tool_parser: {tool_parser}\n" f"Supported parsers: {list(ToolParserFactory.parsers.keys())}" ) from e + parser = tool_parser_class() + parser.grammar_constrained = grammar_constrained + return parser diff --git a/tests/unittest/llmapi/apps/data/nemotron_super_vl_chat_template.jinja b/tests/unittest/llmapi/apps/data/nemotron_super_vl_chat_template.jinja new file mode 100644 index 000000000000..1aa27fc23542 --- /dev/null +++ b/tests/unittest/llmapi/apps/data/nemotron_super_vl_chat_template.jinja @@ -0,0 +1,251 @@ +{# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +Tool-call layout fixture from NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16, +chat_template.jinja revision 29dc69d. +#} +{% macro render_extra_keys(json_dict, handled_keys) %} + {%- if json_dict is mapping %} + {%- for json_key in json_dict if json_key not in handled_keys %} + {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %} + {{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '' }} + {%- else %} + {{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '' }} + {%- endif %} + {%- endfor %} + {%- endif %} +{% endmacro %} +{%- macro render_content(content) -%} + {%- if content is none -%} + {{- '' -}} + {%- elif content is string -%} + {{- content | string -}} + {%- else -%} + {%- set text_ns = namespace(val='') -%} + {%- set counters = namespace(images=0, videos=0) -%} + {%- for part in content -%} + {%- if part['type'] == 'image' or part['type'] == 'image_url' or part['type'] == 'input_image' -%} + {%- set counters.images = counters.images + 1 -%} + {%- elif part['type'] == 'video' or part['type'] == 'video_url' or part['type'] == 'input_video' -%} + {%- set counters.videos = counters.videos + 1 -%} + {%- elif part['type'] == 'text' or part['type'] == 'input_text' -%} + {%- set text_ns.val = text_ns.val + (part['text'] | default('', true)) -%} + {%- endif -%} + {%- endfor -%} + {%- if '' in text_ns.val -%} + {%- set counters.images = 0 -%} + {%- endif -%} + {%- if '"]) +def test_qwen_coder_recovers_unclosed_parameters_before_done(chunk_size, second_call): + from tensorrt_llm.serve.openai_protocol import ResponsesRequest + + text = "abc" + second_call + request = ResponsesRequest( + model="test", + input="Hello", + tools=[ + { + "type": "function", + "name": "f", + "parameters": {"type": "object", "properties": {"x": {"type": "string"}}}, + }, + {"type": "function", "name": "g", "parameters": {"type": "object"}}, + ], + ) + helper = ResponsesStreamingEventsHelper() + parsers = {} + events = [] + for start in range(0, len(text), chunk_size): + end = min(start + chunk_size, len(text)) + events.extend( + _generate_streaming_event( + _make_mock_output(text=text[:end], text_diff=text[start:end]), + request, + False, + helper, + tool_parser_id="qwen3_coder", + tool_parser_dict=parsers, + ) + ) + events.extend( + _generate_streaming_event( + _make_mock_output(text=text), + request, + True, + helper, + tool_parser_id="qwen3_coder", + tool_parser_dict=parsers, + ) + ) + done_calls = [ + event.item + for event in events + if event.type == "response.output_item.done" and event.item.type == "function_call" + ] + expected = [("f", {"x": "abc"})] + ([("g", {})] if second_call else []) + assert [(call.name, json.loads(call.arguments)) for call in done_calls] == expected + added_calls = [ + event.item + for event in events + if event.type == "response.output_item.added" and event.item.type == "function_call" + ] + assert [(call.name, json.loads(call.arguments)) for call in added_calls] == expected + assert [call.id for call in added_calls] == [call.id for call in done_calls] + assert not any(event.type == "response.output_text.delta" for event in events) + + +def test_qwen_coder_partial_preview_preserves_literal_outer_closers(): + from tensorrt_llm.serve.openai_protocol import ResponsesRequest + + request = ResponsesRequest( + model="test", + input="Hello", + tools=[ + { + "type": "function", + "name": "f", + "parameters": {"type": "object", "properties": {"x": {"type": "string"}}}, + } + ], + ) + prefix = "beforea" + text = prefix + "b" + helper = ResponsesStreamingEventsHelper() + parsers = {} + events = list( + _generate_streaming_event( + _make_mock_output(text=prefix, text_diff=prefix), + request, + False, + helper, + tool_parser_id="qwen3_coder", + tool_parser_dict=parsers, + ) + ) + assert not any(event.type == "response.output_text.done" for event in events) + events.extend( + _generate_streaming_event( + _make_mock_output(text=text, text_diff=text[len(prefix) :]), + request, + True, + helper, + tool_parser_id="qwen3_coder", + tool_parser_dict=parsers, + ) + ) + done_calls = [ + event.item + for event in events + if event.type == "response.output_item.done" and event.item.type == "function_call" + ] + assert len(done_calls) == 1 + assert json.loads(done_calls[0].arguments) == {"x": "ab"} diff --git a/tests/unittest/llmapi/apps/test_serving_extensions.py b/tests/unittest/llmapi/apps/test_serving_extensions.py index 777f5da24f47..51b7bea0172c 100644 --- a/tests/unittest/llmapi/apps/test_serving_extensions.py +++ b/tests/unittest/llmapi/apps/test_serving_extensions.py @@ -24,12 +24,29 @@ ServingExtension, apply_model_chat_extensions, register_serving_extension, + required_tool_choice_mode, structured_output_format_for, ) pytestmark = pytest.mark.cpu_only +@pytest.mark.parametrize( + "model_type,expected", + [ + ("kimi_k3", "template"), + ("nemotron_h_omni", "grammar"), + ("nemotron_h", "grammar"), + ("qwen3", None), + ("qwen3_next", None), + ("unknown", None), + (None, None), + ], +) +def test_required_tool_choice_mode(model_type, expected): + assert required_tool_choice_mode(model_type) == expected + + @pytest.fixture def scratch_extension(): """Register a recording extension under unique keys; unregister after.""" @@ -65,6 +82,11 @@ def test_unregistered_model_type_is_a_no_op(self, scratch_extension, model_type) assert scratch_extension.cls.applied_requests == [] +class TestRequiredToolChoiceDispatch: + def test_registered_extension_defaults_to_unsupported(self, scratch_extension) -> None: + assert required_tool_choice_mode(scratch_extension.model_key) is None + + class TestStructuredOutputDispatch: def test_registered_parser_returns_bound_hook(self, scratch_extension) -> None: hook = structured_output_format_for(scratch_extension.parser_key) @@ -122,7 +144,7 @@ class RawGrammarExtension(ServingExtension): class TestBuiltinExtensions: - """The in-tree Kimi K3 and gpt-oss extensions resolve through the registry.""" + """Built-in extensions resolve through the registry.""" @pytest.fixture def clean_registry(self, monkeypatch): @@ -152,6 +174,13 @@ def test_kimi_k3_resolves_to_kimi_extension(self, clean_registry) -> None: assert isinstance(hook.__self__, KimiK3ServingExtension) assert isinstance(serving_extensions._BY_MODEL_TYPE["kimi_k3"], KimiK3ServingExtension) + @pytest.mark.parametrize( + "model_type,expected", + [("kimi_k3", "template"), ("nemotron_h", "grammar"), ("nemotron_h_omni", "grammar")], + ) + def test_required_tool_choice_loads_builtins(self, clean_registry, model_type, expected): + assert required_tool_choice_mode(model_type) == expected + def test_gpt_oss_resolves_to_gpt_oss_extension(self, clean_registry) -> None: hook = structured_output_format_for("gpt_oss") @@ -321,6 +350,7 @@ async def fake_apply_chat_template(**kwargs) -> str: ) def generate_async(*, inputs, **kwargs): + rendered["generation"] = kwargs return SimpleNamespace(prompt_token_ids=[1, 2, 3], finished=True) server = object.__new__(OpenAIServer) @@ -335,6 +365,7 @@ def generate_async(*, inputs, **kwargs): server.tool_parser = None server.tool_call_id_type = "random" server.multimodal_server_config = None + server._input_proc_executor = None server.generator = SimpleNamespace( args=SimpleNamespace( gather_generation_logits=False, @@ -364,6 +395,7 @@ def generate_async(*, inputs, **kwargs): rendered["create_chat_response"] = server._create_chat_response app = FastAPI() + app.state.server = server app.add_api_route("/v1/chat/completions", server.openai_chat, methods=["POST"]) return TestClient(app), events, rendered @@ -438,6 +470,272 @@ def prompt_tokens_excluded_from_usage(self, request) -> int: postproc_params = rendered["create_chat_response"].call_args.args[1] assert postproc_params.postproc_args.num_prompt_tokens_offset == 5 + @pytest.fixture(params=["nemotron_h_omni", "nemotron_h"]) + def required_chat_client(self, chat_client, monkeypatch, request): + from tensorrt_llm.serve.render import chat as render_chat_module + + client, events, rendered = chat_client + server = client.app.state.server + type(server.model_config).model_type = request.param + server.tool_parser = "qwen3_coder" + server.generator.args.reasoning_parser = "nemotron-v3" + server.generator.args.guided_decoding_backend = "xgrammar" + + async def render_prompt(**kwargs): + thinking = rendered.get("thinking", True) + return "<|im_start|>assistant\n" + ("\n" if thinking else "") + + monkeypatch.setattr(render_chat_module, "async_apply_chat_template", render_prompt) + return client, server, rendered + + @staticmethod + def required_body(): + return { + "model": "test-model", + "messages": [{"role": "user", "content": "Say hello."}], + "tools": [{"type": "function", "function": {"name": "ping"}}], + "tool_choice": "required", + "max_tokens": 64, + } + + def test_required_template_mode_uses_kimi_extension(self, chat_client, monkeypatch): + from tensorrt_llm.serve import chat_utils + + # Keep text-only routing independent of multimodal model initialization. + monkeypatch.setattr( + chat_utils.MULTIMODAL_PLACEHOLDER_REGISTRY, "get_content_format", lambda _: None + ) + client, _, rendered = chat_client + server = client.app.state.server + type(server.model_config).model_type = "kimi_k3" + server.tool_parser = "kimi_k3" + server.generator.args.reasoning_parser = "kimi_k3" + server.generator.args.guided_decoding_backend = None + + response = client.post("/v1/chat/completions", json=self.required_body()) + + assert response.status_code == 200, response.text + assert rendered["chat_template_kwargs"]["tool_choice"] == "required" + assert rendered["generation"]["sampling_params"].guided_decoding is None + postproc = server._create_chat_response.call_args.args[1].postproc_args + assert postproc.tool_parser_grammar_constrained is False + + def test_required_grammar_admission_uses_extension_hook(self, required_chat_client): + client, server, rendered = required_chat_client + model_type = "required-grammar-test-model" + type(server.model_config).model_type = model_type + + @register_serving_extension(model_types=(model_type,)) + class RequiredGrammarExtension(ServingExtension): + def required_tool_choice_mode(self): + return "grammar" + + try: + response = client.post("/v1/chat/completions", json=self.required_body()) + finally: + serving_extensions._BY_MODEL_TYPE.pop(model_type, None) + + assert response.status_code == 200, response.text + assert rendered["generation"]["sampling_params"].guided_decoding is not None + + @pytest.mark.parametrize("thinking", [False, True]) + @pytest.mark.parametrize("workers", [0, 4]) + def test_required_attaches_mandatory_grammar(self, required_chat_client, thinking, workers): + import xgrammar + + client, server, rendered = required_chat_client + rendered["thinking"] = thinking + server.generator.args.num_postprocess_workers = workers + body = self.required_body() + body["chat_template_kwargs"] = {"enable_thinking": not thinking} + response = client.post("/v1/chat/completions", json=body) + assert response.status_code == 200, response.text + params = rendered["generation"]["sampling_params"].guided_decoding + assert params is not None + compiler = xgrammar.GrammarCompiler( + xgrammar.TokenizerInfo( + [bytes([i]) for i in range(256)] + [b""], stop_token_ids=[256] + ) + ) + compiled = compiler.compile_structural_tag(params.structural_tag) + matcher = xgrammar.GrammarMatcher(compiled) + if thinking: + assert matcher.accept_string("Considering.") + assert not matcher.accept_token(256) + assert matcher.accept_string("\n\n\n") + assert matcher.accept_token(256) + postproc = server._create_chat_response.call_args.args[1].postproc_args + assert postproc.chat_template_kwargs["enable_thinking"] is thinking + assert postproc.tool_parser_grammar_constrained is True + + @pytest.mark.parametrize( + "change", + [ + {"tools": []}, + {"prompt_token_ids": [1, 2]}, + {"prompt_token_ids_b64": "AQAAAA=="}, + {"add_generation_prompt": False}, + {"disaggregated_params": {"request_type": "context_only"}}, + {"response_format": {"type": "json_object"}}, + ], + ) + def test_required_rejects_conflicting_requests(self, required_chat_client, change): + client, server, rendered = required_chat_client + response = client.post("/v1/chat/completions", json={**self.required_body(), **change}) + # The bare app uses FastAPI's 422 for request-schema errors. + expected_status = 422 if change.get("tools") == [] else 400 + assert response.status_code == expected_status, response.text + assert "generation" not in rendered + + @pytest.mark.parametrize( + "model_type,parser,backend", + [ + ("qwen3", "qwen3_coder", "xgrammar"), + ("qwen3_moe", "qwen3_coder", "xgrammar"), + ("qwen3_next", "qwen3_coder", "xgrammar"), + ("nemotron_h_omni", "qwen3", "xgrammar"), + ("nemotron_h_omni", None, "xgrammar"), + ("nemotron_h_omni", "qwen3_coder", None), + ("nemotron_h_omni", "qwen3_coder", "llguidance"), + ], + ) + def test_required_rejects_unsupported_configuration( + self, required_chat_client, model_type, parser, backend + ): + client, server, rendered = required_chat_client + type(server.model_config).model_type = model_type + server.tool_parser = parser + server.generator.args.guided_decoding_backend = backend + response = client.post("/v1/chat/completions", json=self.required_body()) + assert response.status_code == 400, response.text + assert "generation" not in rendered + + @pytest.mark.parametrize( + "schema", + [ + {"type": "object", "properties": {}, "additionalProperties": True}, + {"type": "object", "properties": {}, "additionalProperties": {}}, + {"type": "object", "required": "x"}, + ], + ) + def test_required_schema_errors_are_client_errors(self, required_chat_client, schema): + client, server, rendered = required_chat_client + body = self.required_body() + body["tools"][0]["function"]["parameters"] = schema + response = client.post("/v1/chat/completions", json=body) + assert response.status_code == 400 + assert "generation" not in rendered + assert "EBNF" not in response.text + assert "json_schema_converter" not in response.text + + @pytest.mark.parametrize( + "reasoning_parser", ["NEMOTRON-V3", "nano-v3", "NANO-V3", "k-exaone", "K-EXAONE"] + ) + def test_required_uses_parser_capabilities( + self, required_chat_client, monkeypatch, reasoning_parser + ): + from tensorrt_llm.serve.render import chat as render_chat_module + from tensorrt_llm.serve.tool_parser.qwen3_coder_parser import Qwen3CoderToolParser + from tensorrt_llm.serve.tool_parser.tool_parser_factory import ToolParserFactory + + client, server, rendered = required_chat_client + monkeypatch.setitem(ToolParserFactory.parsers, "custom_xml", Qwen3CoderToolParser) + server.tool_parser = "CUSTOM_XML" + server.generator.args.reasoning_parser = reasoning_parser + + async def render_prompt(**kwargs): + return "\n" + + monkeypatch.setattr(render_chat_module, "async_apply_chat_template", render_prompt) + response = client.post("/v1/chat/completions", json=self.required_body()) + assert response.status_code == 200, response.text + assert rendered["generation"]["sampling_params"].guided_decoding is not None + args = server._create_chat_response.call_args.args[1].postproc_args + assert args.chat_template_kwargs["enable_thinking"] is False + + @pytest.mark.parametrize("reasoning_parser", [None, "deepseek-r1"]) + def test_required_reports_missing_reasoning_capability( + self, required_chat_client, monkeypatch, reasoning_parser + ): + from unittest.mock import AsyncMock + + from tensorrt_llm.serve.render import chat as render_chat_module + + client, server, rendered = required_chat_client + server.generator.args.reasoning_parser = reasoning_parser + render_prompt = AsyncMock() + monkeypatch.setattr(render_chat_module, "async_apply_chat_template", render_prompt) + + response = client.post("/v1/chat/completions", json=self.required_body()) + + assert response.status_code == 400, response.text + assert "need a reasoning parser that resolves its mode" in response.text + assert "--reasoning_parser nemotron-v3" in response.text + render_prompt.assert_not_called() + assert "generation" not in rendered + + def test_required_rejects_unresolved_prompt(self, required_chat_client, monkeypatch): + from unittest.mock import AsyncMock + + from tensorrt_llm.serve.render import chat as render_chat_module + + client, server, rendered = required_chat_client + monkeypatch.setattr( + render_chat_module, "async_apply_chat_template", AsyncMock(return_value="assistant:") + ) + response = client.post("/v1/chat/completions", json=self.required_body()) + assert response.status_code == 400, response.text + assert ( + "Cannot resolve required-tool reasoning state from the rendered prompt" in response.text + ) + assert "--reasoning_parser" not in response.text + assert "generation" not in rendered + + @pytest.mark.parametrize("reasoning_parser", ["nemotron-v3", "nano-v3", "k-exaone"]) + @pytest.mark.parametrize("thinking", [False, True]) + @pytest.mark.parametrize("tool_choice", [None, "auto"]) + def test_non_required_chat_resolves_mode_from_prompt( + self, chat_client, monkeypatch, reasoning_parser, thinking, tool_choice + ): + from unittest.mock import AsyncMock + + from tensorrt_llm.serve.render import chat as render_chat_module + + client, _, rendered = chat_client + server = client.app.state.server + server.generator.args.reasoning_parser = reasoning_parser + prompt = "<|im_start|>assistant\n" + ("\n" if thinking else "") + monkeypatch.setattr( + render_chat_module, "async_apply_chat_template", AsyncMock(return_value=prompt) + ) + body = { + "model": "test-model", + "messages": [{"role": "user", "content": "Say hello."}], + "chat_template_kwargs": {"thinking": not thinking, "enable_thinking": not thinking}, + } + if tool_choice is not None: + body.update(tools=self.required_body()["tools"], tool_choice=tool_choice) + server.tool_parser = "qwen3_coder" + + response = client.post("/v1/chat/completions", json=body) + + assert response.status_code == 200, response.text + assert rendered["generation"]["sampling_params"].guided_decoding is None + postproc = server._create_chat_response.call_args.args[1].postproc_args + assert postproc.chat_template_kwargs["thinking"] is thinking + assert postproc.chat_template_kwargs["enable_thinking"] is thinking + assert postproc.tool_parser_grammar_constrained is False + + def test_auto_does_not_attach_required_grammar(self, required_chat_client): + client, server, rendered = required_chat_client + response = client.post( + "/v1/chat/completions", json={**self.required_body(), "tool_choice": "auto"} + ) + assert response.status_code == 200, response.text + assert rendered["generation"]["sampling_params"].guided_decoding is None + postproc = server._create_chat_response.call_args.args[1].postproc_args + assert postproc.tool_parser_grammar_constrained is False + def _chat_request(**overrides): from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest diff --git a/tests/unittest/llmapi/apps/test_tool_parsers.py b/tests/unittest/llmapi/apps/test_tool_parsers.py index defb79ac8312..24b8e77cd0a8 100644 --- a/tests/unittest/llmapi/apps/test_tool_parsers.py +++ b/tests/unittest/llmapi/apps/test_tool_parsers.py @@ -1142,6 +1142,631 @@ def test_streaming_zero_arg_tool(self, parser, arguments_chunk): assert "" not in parser._buffer +class TestQwen3CoderRequiredTools: + + @staticmethod + def _call(name, parameter="", value=""): + body = f"{value}\n" if parameter else "" + return f"\n\n{body}\n" + + @pytest.fixture + def compiler(self): + import xgrammar + + tokenizer = xgrammar.TokenizerInfo([bytes([i]) + for i in range(256)] + [b""], + stop_token_ids=[256]) + return xgrammar.GrammarCompiler(tokenizer) + + @pytest.mark.parametrize("thinking", [False, True]) + def test_required_grammar_accepts_model_template(self, compiler, thinking): + from pathlib import Path + + import xgrammar + from jinja2 import Environment + + tools = [ + ChatCompletionToolsParam(type="function", + function=FunctionDefinition( + name="f", + parameters={ + "type": "object", + "properties": { + "location": { + "type": "string" + }, + "unit": { + "type": "string" + } + }, + "required": ["location"] + })) + ] + template = Environment().from_string( + (Path(__file__).parent / "data" / + "nemotron_super_vl_chat_template.jinja").read_text()) + rendered = template.render( + messages=[{ + "role": "user", + "content": "Weather?" + }, { + "role": + "assistant", + "content": + "", + "reasoning_content": + "Checking." if thinking else "", + "tool_calls": [{ + "function": { + "name": "f", + "arguments": { + "location": "Denver", + "unit": "celsius" + } + } + }] + }], + tools=[tool.model_dump(exclude_none=True) for tool in tools], + add_generation_prompt=False) + output = rendered.rsplit("<|im_start|>assistant\n", + 1)[1].split("<|im_end|>", 1)[0] + prefill = "" if thinking else "" + assert output.startswith(prefill) + output = output[len(prefill):] + matcher = xgrammar.GrammarMatcher( + compiler.compile_structural_tag( + Qwen3CoderToolParser().build_required_structural_tag( + tools, thinking=thinking))) + assert matcher.accept_string(output) and matcher.accept_token(256) + + @pytest.mark.parametrize("thinking", [False, True]) + @pytest.mark.parametrize("required", + [["location"], ["location", "unit"], []]) + def test_required_grammar_accepts_parameter_whitespace( + self, compiler, thinking, required): + import xgrammar + + tools = [ + ChatCompletionToolsParam(type="function", + function=FunctionDefinition( + name="get_weather", + parameters={ + "type": "object", + "properties": { + "location": { + "type": "string" + }, + "unit": { + "type": "string" + } + }, + "required": required, + "additionalProperties": False + })) + ] + fmt = Qwen3CoderToolParser().build_required_structural_tag( + tools, thinking=thinking) + compiled = compiler.compile_structural_tag(fmt) + prefix = "Checking." if thinking else "" + for whitespace in ("", "\n", "\n\t ", "\r\n"): + body = ("Denver" + whitespace + + "celsius" + whitespace) + text = prefix + whitespace + "\n\n" + body + "\n" + matcher = xgrammar.GrammarMatcher(compiled) + assert matcher.accept_string(text) + assert matcher.accept_token(256) + parsed = Qwen3CoderToolParser().detect_and_parse( + text[len(prefix):], tools) + assert json.loads(parsed.calls[0].parameters) == { + "location": "Denver", + "unit": "celsius" + } + for body, accepted in [("", not required), + ("Denver\n", + "unit" not in required)]: + matcher = xgrammar.GrammarMatcher(compiled) + text = prefix + "\n\n\n" + body + "\n" + assert (matcher.accept_string(text) + and matcher.accept_token(256)) is accepted + matcher = xgrammar.GrammarMatcher(compiled) + assert matcher.accept_string(prefix + "\n ") + assert not matcher.accept_token(256) + + @pytest.mark.parametrize("schema", [ + { + "type": "object", + "properties": {}, + "additionalProperties": True + }, + { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + { + "type": "object", + "properties": {}, + "additionalProperties": { + "type": "string" + } + }, + { + "type": "object", + "required": "x" + }, + { + "type": "object", + "minProperties": "bad" + }, + { + "type": "object", + "properties": {}, + "required": ["x"] + }, + { + "type": "object", + "properties": {}, + "minProperties": 1 + }, + ]) + def test_required_grammar_rejects_unsupported_schemas(self, schema): + tools = [ + ChatCompletionToolsParam(type="function", + function=FunctionDefinition( + name="f", parameters=schema)) + ] + with pytest.raises(ValueError) as exc: + Qwen3CoderToolParser().build_required_structural_tag(tools, + thinking=False) + assert "EBNF" not in str(exc.value) + + @pytest.mark.parametrize( + "value", + ["ab", "ab", "ab", "ab"]) + @pytest.mark.parametrize("chunk_size", [1, 7, 10000]) + def test_grammar_accepted_delimiters_preserve_arguments( + self, compiler, value, chunk_size): + import xgrammar + + tools = [ + ChatCompletionToolsParam(type="function", + function=FunctionDefinition( + name="f", + parameters={ + "type": "object", + "properties": { + "x": { + "type": "string" + } + }, + "required": ["x"] + })) + ] + text = self._call("f", "x", value) + "\n" + self._call( + "f", "x", "second") + parser = Qwen3CoderToolParser() + matcher = xgrammar.GrammarMatcher( + compiler.compile_structural_tag( + parser.build_required_structural_tag(tools, thinking=False))) + assert matcher.accept_string(text) and matcher.accept_token(256) + whole = parser.detect_and_parse(text, tools) + assert [json.loads(call.parameters) for call in whole.calls] == [{ + "x": + value + }, { + "x": + "second" + }] + results = [ + parser.parse_streaming_increment(text[i:i + chunk_size], tools) + for i in range(0, len(text), chunk_size) + ] + [parser.finish(tools)] + arguments = {} + for result in results: + for call in result.calls: + arguments[call.tool_index] = arguments.get(call.tool_index, + "") + call.parameters + assert [json.loads(arguments[i]) for i in range(2)] == [{ + "x": value + }, { + "x": "second" + }] + assert "".join(result.normal_text for result in results) == "\n" + + def test_incomplete_function_header_is_not_a_call(self, sample_tools): + text = "" + parser = Qwen3CoderToolParser() + assert parser.detect_and_parse(text, sample_tools).calls == [] + assert parser.parse_streaming_increment(text, sample_tools).calls == [] + assert parser.finish(sample_tools).calls == [] + + @pytest.mark.timeout(3) + @pytest.mark.parametrize("name", ["", " ", "\t"]) + @pytest.mark.parametrize("chunk_size", [1, 7, 10000]) + @pytest.mark.parametrize("body", [ + "", + "ab", + ]) + def test_empty_function_name_recovers( + self, sample_tools: list[ChatCompletionToolsParam], name: str, + chunk_size: int, body: str) -> None: + text = (f"{body}" + + self._call("get_weather", "location", "Denver")) + parser = Qwen3CoderToolParser() + results = [ + parser.parse_streaming_increment(text[i:i + chunk_size], + sample_tools) + for i in range(0, len(text), chunk_size) + ] + [parser.finish(sample_tools)] + calls = [call for result in results for call in result.calls] + assert "".join(result.normal_text for result in results) == "" + assert [call.name for call in calls if call.name] == ["get_weather"] + assert {call.tool_index for call in calls} == {0} + assert json.loads("".join(call.parameters for call in calls)) == { + "location": "Denver" + } + + @pytest.mark.parametrize("chunk_size", [1, 7, 10000]) + def test_empty_function_name_preserves_surrounding_output( + self, sample_tools: list[ChatCompletionToolsParam], + chunk_size: int) -> None: + text = ("before" + self._call("get_weather", "location", "Taipei") + + "between" + + self._call("get_weather", "location", "Denver") + "after") + parser = Qwen3CoderToolParser() + results = [ + parser.parse_streaming_increment(text[i:i + chunk_size], + sample_tools) + for i in range(0, len(text), chunk_size) + ] + [parser.finish(sample_tools)] + calls = [call for result in results for call in result.calls] + assert "".join(result.normal_text + for result in results) == "beforebetweenafter" + assert [(call.tool_index, call.name) for call in calls + if call.name] == [(0, "get_weather"), (1, "get_weather")] + for index, location in enumerate(["Taipei", "Denver"]): + assert json.loads("".join(call.parameters for call in calls + if call.tool_index == index)) == { + "location": location + } + + @pytest.mark.parametrize("body", [ + "", + "ab", + "Denver None: + parser = Qwen3CoderToolParser() + result = parser.parse_streaming_increment( + "" + body, sample_tools) + assert result.normal_text == "" + assert result.calls == [] + tail = parser.finish(sample_tools) + assert tail.normal_text == "" + assert tail.calls == [] + + @pytest.mark.parametrize("thinking", [False, True]) + def test_required_grammar_rejects_empty_and_unknown_calls( + self, sample_tools, compiler, thinking): + import xgrammar + + fmt = Qwen3CoderToolParser().build_required_structural_tag( + sample_tools, thinking=thinking) + compiled = compiler.compile_structural_tag(fmt) + prefix = "Check the available tools." if thinking else "" + weather = self._call("get_weather", "location", "Denver") + search = self._call("search_web", "query", "weather") + for text, expected in [ + ("", False), + (prefix, False), + (prefix + "Hello.", False), + (prefix + weather, True), + (prefix + weather + "\n", True), + (prefix + weather + "\n\n", True), + (prefix + search, True), + (prefix + weather + "\n" + search, True), + (prefix + weather + "\n\n" + search, True), + (prefix + self._call("unknown", "location", "Denver"), False), + (prefix + weather + "\n" + + self._call("unknown", "location", "Denver"), False), + (prefix + self._call("get_weather"), False), + (prefix + weather[:-12], False), + ]: + matcher = xgrammar.GrammarMatcher(compiled) + accepted = matcher.accept_string(text) and matcher.accept_token(256) + assert accepted == expected, text + if thinking: + matcher = xgrammar.GrammarMatcher(compiled) + assert not matcher.accept_string("Consider ") + + @pytest.mark.parametrize("parameters", [ + None, {}, { + "type": "object" + }, { + "type": "object", + "properties": {}, + "additionalProperties": False + } + ]) + def test_required_grammar_supports_parameterless_tools( + self, compiler, parameters): + import xgrammar + + tools = [ + ChatCompletionToolsParam(type="function", + function=FunctionDefinition( + name="ping", parameters=parameters)) + ] + fmt = Qwen3CoderToolParser().build_required_structural_tag( + tools, thinking=False) + matcher = xgrammar.GrammarMatcher(compiler.compile_structural_tag(fmt)) + assert matcher.accept_string(self._call("ping")) + assert matcher.accept_token(256) + assert tools[0].function.parameters == parameters + + @pytest.mark.parametrize("parameters,value", [ + ({ + "type": "object", + "properties": { + "x": { + "type": "array", + "items": { + "type": "integer" + } + } + }, + "required": ["x"] + }, "[1,2]"), + ({ + "type": "object", + "properties": { + "x": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + } + } + } + }, + "required": ["x"] + }, '{"ok":true}'), + ({ + "type": "object", + "properties": { + "x": { + "anyOf": [{ + "type": "string" + }, { + "type": "null" + }] + } + }, + "required": ["x"] + }, "null"), + ]) + def test_required_grammar_supports_nested_and_nullable_arguments( + self, compiler, parameters, value): + import xgrammar + + tools = [ + ChatCompletionToolsParam(type="function", + function=FunctionDefinition( + name="f", parameters=parameters)) + ] + fmt = Qwen3CoderToolParser().build_required_structural_tag( + tools, thinking=False) + matcher = xgrammar.GrammarMatcher(compiler.compile_structural_tag(fmt)) + text = self._call("f", "x", value) + assert matcher.accept_string(text) + assert matcher.accept_token(256) + assert json.loads(Qwen3CoderToolParser().detect_and_parse( + text, tools).calls[0].parameters) == { + "x": json.loads(value) + } + + @pytest.mark.parametrize("thinking", [False, True]) + @pytest.mark.parametrize("chunk_size", [1, 7, 10000]) + def test_required_stream_matches_complete_response(self, sample_tools, + thinking, chunk_size): + from tensorrt_llm.serve.postprocess_handlers import \ + chat_response_post_processor + + tools = sample_tools + [ + ChatCompletionToolsParam(type="function", + function=FunctionDefinition(name="ping")) + ] + text = ("Checking." if thinking else "") + "\n".join([ + self._call("get_weather", "location", "}"), + self._call("search_web", "query", "weather"), + self._call("ping") + ]) + + def make_args(): + args = TestForcedToolChoicePostprocessing._make_args( + tools, tool_parser="qwen3_coder") + args.tool_choice = "required" + args.tool_parser_grammar_constrained = True + args.reasoning_parser = "nemotron-v3" + args.chat_template_kwargs = {"enable_thinking": thinking} + return args + + whole = chat_response_post_processor( + TestForcedToolChoicePostprocessing._fake_response(text), + make_args()) + chunks = [ + text[i:i + chunk_size] for i in range(0, len(text), chunk_size) + ] + streamed = TestForcedToolChoicePostprocessing._stream_deltas( + make_args(), chunks) + calls = {} + reasoning = "" + for payload in streamed: + for choice in payload["choices"]: + delta = choice["delta"] + reasoning += delta.get("reasoning_content") or "" + for call in delta.get("tool_calls") or []: + item = calls.setdefault(call["index"], { + "name": "", + "arguments": "", + "ids": [] + }) + item["name"] += call["function"].get("name") or "" + item["arguments"] += call["function"].get("arguments") or "" + if call.get("id"): + item["ids"].append(call["id"]) + expected = whole.choices[0] + assert len(calls) == len(expected.message.tool_calls) == 3 + for index, call in enumerate(expected.message.tool_calls): + assert calls[index]["name"] == call.function.name + assert json.loads(calls[index]["arguments"]) == json.loads( + call.function.arguments) + assert len(calls[index]["ids"]) == 1 + assert reasoning == (expected.message.reasoning_content or "") + assert streamed[-1]["choices"][0][ + "finish_reason"] == expected.finish_reason == "tool_calls" + + def test_required_truncation_preserves_finish_reason(self, sample_tools): + from tensorrt_llm.serve.postprocess_handlers import \ + chat_response_post_processor + + text = self._call("get_weather", "location", "Denver")[:-12] + args = TestForcedToolChoicePostprocessing._make_args( + sample_tools, tool_parser="qwen3_coder") + args.tool_choice = "required" + whole = chat_response_post_processor( + TestForcedToolChoicePostprocessing._fake_response(text, "length"), + args) + assert whole.choices[0].finish_reason == "length" + assert not whole.choices[0].message.tool_calls + streamed = TestForcedToolChoicePostprocessing._stream_deltas( + args, [text], "length") + assert streamed[-1]["choices"][0]["finish_reason"] == "length" + + +class TestQwen3CoderParameterRecovery: + FIRST = "abc" + SECOND = "1" + + @pytest.fixture + def tools(self): + return [ + ChatCompletionToolsParam( + type="function", + function=FunctionDefinition( + name=name, + parameters={ + "type": "object", + "properties": { + parameter: { + "type": "string" + } + }, + }, + ), + ) for name, parameter in [("f", "x"), ("g", "y")] + ] + + @pytest.mark.parametrize("chunk_size", [1, 7, 10000]) + @pytest.mark.parametrize("suffix,expected", [ + ("", [("f", { + "x": "abc" + })]), + (SECOND, [("f", { + "x": "abc" + }), ("g", { + "y": "1" + })]), + ("", [("f", { + "x": "abc" + }), ("g", {})]), + ]) + def test_missing_parameter_closer_recovers_calls(self, tools, chunk_size, + suffix, expected): + text = self.FIRST + suffix + parser = Qwen3CoderToolParser() + whole = parser.detect_and_parse(text, tools) + assert whole.normal_text == "" + assert [(call.name, json.loads(call.parameters)) + for call in whole.calls] == expected + + results = [ + parser.parse_streaming_increment(text[i:i + chunk_size], tools) + for i in range(0, len(text), chunk_size) + ] + [parser.finish(tools)] + calls = {} + for result in results: + for call in result.calls: + item = calls.setdefault(call.tool_index, { + "name": "", + "arguments": "" + }) + item["name"] += call.name or "" + item["arguments"] += call.parameters + assert [(item["name"], json.loads(item["arguments"])) + for item in calls.values()] == expected + assert "".join(result.normal_text for result in results) == "" + assert parser.finish(tools).calls == [] + + @pytest.mark.parametrize("constrained", [False, True]) + @pytest.mark.parametrize("chunk_size", [1, 7, 10000]) + def test_chat_recovery_policy_preserves_required_arguments( + self, tools, constrained, chunk_size): + import xgrammar + + from tensorrt_llm.serve.postprocess_handlers import \ + chat_response_post_processor + + value = "abc\n\n\n\n1" + text = "\n\n" + value + "\n\n" + compiler = xgrammar.GrammarCompiler( + xgrammar.TokenizerInfo([bytes([i]) + for i in range(256)] + [b""], + stop_token_ids=[256])) + matcher = xgrammar.GrammarMatcher( + compiler.compile_structural_tag( + Qwen3CoderToolParser().build_required_structural_tag( + tools, thinking=False))) + assert matcher.accept_string(text) and matcher.accept_token(256) + + def make_args(): + args = TestForcedToolChoicePostprocessing._make_args( + tools, tool_parser="qwen3_coder") + args.tool_choice = "required" if constrained else "auto" + args.tool_parser_grammar_constrained = constrained + return args + + expected = [("f", { + "x": value + })] if constrained else [("f", { + "x": "abc" + }), ("g", { + "y": "1" + })] + whole = chat_response_post_processor( + TestForcedToolChoicePostprocessing._fake_response(text), + make_args()) + assert [(call.function.name, json.loads(call.function.arguments)) + for call in whole.choices[0].message.tool_calls] == expected + payloads = TestForcedToolChoicePostprocessing._stream_deltas( + make_args(), + [text[i:i + chunk_size] for i in range(0, len(text), chunk_size)]) + calls = {} + for payload in payloads: + for choice in payload["choices"]: + for call in choice["delta"].get("tool_calls") or []: + item = calls.setdefault(call["index"], { + "name": "", + "arguments": "" + }) + item["name"] += call["function"].get("name") or "" + item["arguments"] += call["function"].get("arguments") or "" + assert [(item["name"], json.loads(item["arguments"])) + for item in calls.values()] == expected + + class TestQwen3CoderToolParser(BaseToolParserTestClass): """Test suite for Qwen3CoderToolParser class.""" diff --git a/tests/unittest/llmapi/test_reasoning_parser.py b/tests/unittest/llmapi/test_reasoning_parser.py index 07c2f547f3d3..28081b53ae15 100644 --- a/tests/unittest/llmapi/test_reasoning_parser.py +++ b/tests/unittest/llmapi/test_reasoning_parser.py @@ -548,15 +548,12 @@ def test_resolved_mode_overrides_whatever_the_caller_sent( @pytest.mark.parametrize("parser", [ "deepseek-r1", "deepseek_v4", "deepseek_v41", "qwen3", "qwen3_5", - "minimax_m2", "minimax_m3", "nemotron-v3", "nano-v3", "gemma4", "kimi_k2", - "kimi_k25" + "minimax_m2", "minimax_m3", "gemma4", "kimi_k2", "kimi_k25" ]) def test_resolve_prefilled_thinking_requires_opt_in(parser: str) -> None: """Parsers that have not opted in must never be resolved from the prompt. - The DeepSeek V4 family shares the base class and `nemotron-v3` / `nano-v3` - also read `enable_thinking`, so without the flag they would silently pick - up a mode the server inferred. + Parsers that derive their mode from request kwargs retain that policy. """ # Otherwise a typo or a dropped registration passes vacuously, since an # unknown name also resolves to None. @@ -1023,12 +1020,10 @@ def test_auto_detect_nemotron_h(tmp_path, model_type): assert result == "nemotron-v3" -def test_nemotron_v3_alias_same_parser(): - """'nemotron-v3' and the legacy 'nano-v3' resolve to the same parser.""" - nemotron = ReasoningParserFactory.create_reasoning_parser("nemotron-v3") - nano = ReasoningParserFactory.create_reasoning_parser("nano-v3") - assert isinstance(nemotron, NemotronV3ReasoningParser) - assert isinstance(nano, NemotronV3ReasoningParser) +@pytest.mark.parametrize("name", ["nemotron-v3", "nano-v3", "k-exaone"]) +def test_shared_reasoning_parser_registration(name): + parser = ReasoningParserFactory.create_reasoning_parser(name) + assert type(parser) is NemotronV3ReasoningParser # --------------------------------------------------------------------------- @@ -1414,6 +1409,52 @@ def test_deepseek_r1_keeps_a_second_close_as_content() -> None: assert result.content == "bf" +@pytest.mark.parametrize( + "parser_name", + ["nemotron-v3", "nano-v3", "NEMOTRON-V3", "k-exaone", "K-EXAONE"]) +@pytest.mark.parametrize("tail,thinking", [ + ("\n", True), + ("", False), + ("\n\n\n\n", False), +]) +def test_shared_reasoning_parser_resolves_prefilled_mode( + parser_name, tail, thinking): + resolved = ReasoningParserFactory.resolve_prefilled_thinking( + parser_name, "" + tail) + assert resolved is thinking + parser = ReasoningParserFactory.create_reasoning_parser( + parser_name, {"enable_thinking": resolved}) + output = "hiddenvisible" if thinking else "visible" + result = parser.parse(output) + assert result.content == "visible" + assert result.reasoning_content == ("hidden" if thinking else "") + + +@pytest.mark.parametrize("chunks,expected_reasoning,expected_content", [ + (["planf" + ], "plan", "f"), + (["planf" + ], "plan", "f"), + (["planab" + ], "plan", "ab"), + (["planf" + ], "plan", "f"), + (["planf"], "plan", "f"), +]) +def test_nemotron_first_boundary_ends_reasoning(chunks, expected_reasoning, + expected_content): + parser = NemotronV3ReasoningParser() + whole = parser.parse("".join(chunks)) + assert whole.reasoning_content == expected_reasoning + assert whole.content == expected_content + results = [parser.parse_delta(chunk) + for chunk in chunks] + [parser.finish()] + assert "".join(result.reasoning_content or "" + for result in results) == expected_reasoning + assert "".join(result.content or "" + for result in results) == expected_content + + M3_START, M3_END = "", "" # How the server relays the mode it resolved from the rendered prompt: # thinking enabled (`` prefilled), disabled (``