Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions src/trinity/llm/openai_compatible_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,15 @@ def _parse_completion(data: dict, model: str) -> ChatResult:
completion (``text=""``, ``finish_reason="error"``), mirroring the existing
null-``content`` handling so a single odd reply degrades one turn instead of
crashing the run. See JOURNAL 2026-07-08.

Reasoning models (e.g. the DeepSeek / GLM / Kimi "thinking" variants served
over OpenAI-compatible APIs) can return the answer in a separate
``reasoning_content`` channel with ``content`` null or empty — most commonly
when the visible channel is cut off by the token cap
(``finish_reason="length"``). When ``content`` carries no text we fall back to
``reasoning_content`` so that answer stays scoreable instead of collapsing the
turn to ``text=""`` (which zeroes the reward). A non-empty ``content`` always
wins, so ordinary completions are unaffected.
"""
usage = data.get("usage") or {}
prompt_tokens = int(usage.get("prompt_tokens", 0) or 0)
Expand All @@ -113,6 +122,8 @@ def _parse_completion(data: dict, model: str) -> ChatResult:
choice = choices[0] or {}
message = choice.get("message") or {}
content = message.get("content")
if content is None or content == "":
content = message.get("reasoning_content")
return ChatResult(
model=model,
text="" if content is None else str(content),
Expand Down
42 changes: 42 additions & 0 deletions tests/test_pool_parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,3 +64,45 @@ def test_non_string_content_is_stringified():
data = {"choices": [{"message": {"content": 42}, "finish_reason": "stop"}]}
res = _parse_completion(data, "m")
assert res.text == "42"


def test_null_content_falls_back_to_reasoning_content():
# Reasoning models can put the answer in reasoning_content with content=null,
# commonly when the visible channel is truncated by the token cap.
data = {
"choices": [
{
"message": {"content": None, "reasoning_content": r"...so \boxed{42}"},
"finish_reason": "length",
}
]
}
res = _parse_completion(data, "m")
assert res.text == r"...so \boxed{42}"
assert res.finish_reason == "length"


def test_empty_content_falls_back_to_reasoning_content():
data = {
"choices": [
{"message": {"content": "", "reasoning_content": "the answer is B"}, "finish_reason": "stop"}
]
}
res = _parse_completion(data, "m")
assert res.text == "the answer is B"


def test_non_empty_content_wins_over_reasoning_content():
data = {
"choices": [
{"message": {"content": "final answer", "reasoning_content": "scratch work"}}
]
}
res = _parse_completion(data, "m")
assert res.text == "final answer"


def test_null_content_without_reasoning_content_stays_empty():
data = {"choices": [{"message": {"content": None}, "finish_reason": "stop"}]}
res = _parse_completion(data, "m")
assert res.text == ""
Loading