Skip to content

Commit eb69413

Browse files
LeizhenZhangLeizhenZhang
committed
[#21004][fix] DeepSeek V3 and V3.1 tool parsers: complete a tool call that ends in the last delta
parse_streaming_increment sends at most one tool name or the arguments of one call per increment, and the arguments follow the name one increment later. When the last delta of a stream holds the closing part of a call (V3: the closing fence and the end tokens, V3.1: the separator, the arguments and the end tokens), the name went out and the arguments were never sent, because finish() was the default no-op. Several tokens per delta happen with speculative decoding and with stream_interval > 1. finish() now sends the rest of the arguments of the open call. It acts only when the name of that call has been sent, the buffer holds exactly one call with its end token, the name in the buffer is the same non-empty name, and the arguments there are a complete JSON object that starts with what was already sent. These are the arguments detect_and_parse reads for that call. In every other case, for example a damaged end token, a call without its end, or two calls that arrived in one delta, finish() sends nothing, as before. Signed-off-by: LeizhenZhang <zhangleizhen645@gmail.com>
1 parent 977a0a9 commit eb69413

3 files changed

Lines changed: 171 additions & 0 deletions

File tree

‎tensorrt_llm/serve/tool_parser/deepseekv31_parser.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,39 @@ def has_tool_call(self, text: str) -> bool:
6969
"""Check if the text contains a deepseek format tool call."""
7070
return self.bot_token in text
7171

72+
def finish(self, tools: List[Tool]) -> StreamingParseResult:
73+
"""Send the arguments of a tool call whose end came in the last delta.
74+
75+
An increment sends at most one tool name or the arguments of one call,
76+
and the arguments go out one increment after the name. So when the last
77+
delta holds the end of a call, its name has been sent and its arguments
78+
have not. They are sent here when the buffer holds only that call, with
79+
its end token, the same non-empty name and a JSON object that continues
80+
what was already sent. In any other case nothing is sent.
81+
"""
82+
func_detail = None
83+
if self.current_tool_name_sent and self._buffer.count("<｜tool▁call▁begin｜>") == 1:
84+
match = re.search(self.func_call_regex, self._buffer, re.DOTALL)
85+
func_detail = match and re.search(self.func_detail_regex, match.group(), re.DOTALL)
86+
if not func_detail:
87+
return StreamingParseResult()
88+
func_name = func_detail.group(1)
89+
func_args = func_detail.group(2).strip()
90+
if (
91+
not func_name
92+
or func_name != self.prev_tool_call_arr[self.current_tool_id]["name"]
93+
or not (func_args.startswith("{") and is_complete_json(func_args))
94+
or not func_args.startswith(self._last_arguments)
95+
or func_args == self._last_arguments
96+
):
97+
return StreamingParseResult()
98+
argument_diff = func_args[len(self._last_arguments) :]
99+
self._last_arguments = func_args
100+
self.streamed_args_for_tool[self.current_tool_id] += argument_diff
101+
return StreamingParseResult(
102+
calls=[ToolCallItem(tool_index=self.current_tool_id, parameters=argument_diff)]
103+
)
104+
72105
def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
73106
"""One-time parsing: Detects and parses tool calls in the provided text.
74107

‎tensorrt_llm/serve/tool_parser/deepseekv3_parser.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,39 @@ def has_tool_call(self, text: str) -> bool:
6969
"""Check if the text contains a deepseek format tool call."""
7070
return self.bot_token in text
7171

72+
def finish(self, tools: List[Tool]) -> StreamingParseResult:
73+
"""Send the arguments of a tool call whose end came in the last delta.
74+
75+
An increment sends at most one tool name or the arguments of one call,
76+
and the arguments go out one increment after the name. So when the last
77+
delta holds the end of a call, its name has been sent and its arguments
78+
have not. They are sent here when the buffer holds only that call, with
79+
its end token, the same non-empty name and a JSON object that continues
80+
what was already sent. In any other case nothing is sent.
81+
"""
82+
func_detail = None
83+
if self.current_tool_name_sent and self._buffer.count("<｜tool▁call▁begin｜>") == 1:
84+
match = re.search(self.func_call_regex, self._buffer, re.DOTALL)
85+
func_detail = match and re.search(self.func_detail_regex, match.group(), re.DOTALL)
86+
if not func_detail:
87+
return StreamingParseResult()
88+
func_name = func_detail.group(2)
89+
func_args = func_detail.group(3).strip()
90+
if (
91+
not func_name
92+
or func_name != self.prev_tool_call_arr[self.current_tool_id]["name"]
93+
or not (func_args.startswith("{") and is_complete_json(func_args))
94+
or not func_args.startswith(self._last_arguments)
95+
or func_args == self._last_arguments
96+
):
97+
return StreamingParseResult()
98+
argument_diff = func_args[len(self._last_arguments) :]
99+
self._last_arguments = func_args
100+
self.streamed_args_for_tool[self.current_tool_id] += argument_diff
101+
return StreamingParseResult(
102+
calls=[ToolCallItem(tool_index=self.current_tool_id, parameters=argument_diff)]
103+
)
104+
72105
def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
73106
"""One-time parsing: Detects and parses tool calls in the provided text.
74107

‎tests/unittest/llmapi/apps/test_tool_parsers.py‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3207,6 +3207,111 @@ def test_dsml_string_parameter_values_are_kept_as_written(
32073207
for call in calls)) == arguments
32083208

32093209

3210+
_DS_V3_CALL = ("<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>"
3211+
'get_weather\n```json\n{"location": "Tokyo"}\n```'
3212+
"<｜tool▁call▁end｜><｜tool▁calls▁end｜>")
3213+
_DS_V31_CALL = ("<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_weather<｜tool▁sep｜>"
3214+
'{"location": "Tokyo"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>')
3215+
3216+
3217+
@pytest.mark.parametrize(
3218+
"parser_cls, deltas",
3219+
[
3220+
# The closing fence and both end tokens arrive in the last delta.
3221+
(DeepSeekV3Parser, [
3222+
"<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>"
3223+
'get_weather\n```json\n{"location": "Tokyo"}',
3224+
"\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"
3225+
]),
3226+
(DeepSeekV3Parser, [_DS_V3_CALL]),
3227+
(DeepSeekV31Parser, [_DS_V31_CALL]),
3228+
# A call without arguments: the separator, the empty arguments and
3229+
# the end tokens arrive together.
3230+
(DeepSeekV31Parser, [
3231+
"<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_weather",
3232+
"<｜tool▁sep｜>{}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"
3233+
]),
3234+
# A stray fence after the call is not part of the arguments.
3235+
(DeepSeekV3Parser, [
3236+
_DS_V3_CALL.replace("<｜tool▁calls▁end｜>",
3237+
"\n```json\n<｜tool▁calls▁end｜>")
3238+
]),
3239+
],
3240+
ids=[
3241+
"v3_fence_in_last_delta", "v3_whole_call", "v31_whole_call",
3242+
"v31_no_arguments", "v3_stray_fence_after_call"
3243+
],
3244+
)
3245+
def test_deepseek_finish_completes_a_call_from_the_last_delta(
3246+
sample_tools: list[ChatCompletionToolsParam],
3247+
parser_cls: type[BaseToolParser], deltas: list[str]) -> None:
3248+
"""finish() completes a tool call whose end came in the last delta.
3249+
3250+
Several tokens per delta (speculative decoding, stream_interval) can put
3251+
the end of a call, or the whole call, in the last delta.
3252+
"""
3253+
parser = parser_cls()
3254+
results = [
3255+
parser.parse_streaming_increment(delta, sample_tools)
3256+
for delta in deltas
3257+
]
3258+
results.append(parser.finish(sample_tools))
3259+
3260+
streamed = {}
3261+
for result in results:
3262+
for call in result.calls:
3263+
name, arguments = streamed.get(call.tool_index, (None, ""))
3264+
streamed[call.tool_index] = (call.name
3265+
or name, arguments + call.parameters)
3266+
whole = parser_cls().detect_and_parse("".join(deltas), sample_tools)
3267+
assert [
3268+
(name, json.loads(arguments)) for name, arguments in streamed.values()
3269+
] == [(call.name, json.loads(call.parameters)) for call in whole.calls]
3270+
3271+
3272+
@pytest.mark.parametrize(
3273+
"parser_cls, deltas",
3274+
[
3275+
# Two calls in one delta: the streamed arguments of the first call
3276+
# already come from the second, so sending the second would repeat it.
3277+
(DeepSeekV3Parser, [
3278+
"<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>"
3279+
'get_weather\n```json\n{"location": "a"}\n```<｜tool▁call▁end｜>'
3280+
"<｜tool▁call▁begin｜>function<｜tool▁sep｜>"
3281+
'get_weather\n```json\n{"loc',
3282+
'ation": "Tokyo"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>'
3283+
]),
3284+
# The second call's separator arrived while the first call was open.
3285+
(DeepSeekV31Parser, [
3286+
"<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_weather<｜tool▁sep｜>"
3287+
'{"location": "a"', ', "unit"',
3288+
': "c"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>get_weather<｜tool▁sep｜>',
3289+
'{"location": "P'
3290+
]),
3291+
# The call has no end token.
3292+
(DeepSeekV3Parser,
3293+
[_DS_V3_CALL.replace("<｜tool▁call▁end｜><｜tool▁calls▁end｜>", "")]),
3294+
(DeepSeekV31Parser,
3295+
[_DS_V31_CALL.replace("<｜tool▁call▁end｜>", "<｜tool▁call▁en")]),
3296+
],
3297+
ids=[
3298+
"v3_two_calls_in_one_delta", "v31_second_call_open", "v3_no_end_token",
3299+
"v31_broken_end_token"
3300+
],
3301+
)
3302+
def test_deepseek_finish_sends_nothing_when_the_buffer_is_ambiguous(
3303+
sample_tools: list[ChatCompletionToolsParam],
3304+
parser_cls: type[BaseToolParser], deltas: list[str]) -> None:
3305+
"""finish() sends nothing unless the buffer holds exactly one whole call."""
3306+
parser = parser_cls()
3307+
for delta in deltas:
3308+
parser.parse_streaming_increment(delta, sample_tools)
3309+
3310+
result = parser.finish(sample_tools)
3311+
3312+
assert result.calls == []
3313+
3314+
32103315
@pytest.mark.parametrize(
32113316
"parser_cls, tool_call_text",
32123317
[

0 commit comments

Comments
 (0)