diff --git a/nodes/src/nodes/llm_anthropic/README.md b/nodes/src/nodes/llm_anthropic/README.md index 5cf5ba9b9..ef23ac9ab 100644 --- a/nodes/src/nodes/llm_anthropic/README.md +++ b/nodes/src/nodes/llm_anthropic/README.md @@ -14,9 +14,10 @@ directly for save-time validation. The configured `modelOutputTokens` is passed the model as `max_tokens`. Token counts for budgeting are estimated at roughly 4 characters per token. -When the selected model is flagged as reasoning-capable, the node enables -extended thinking automatically and streams the model's reasoning over the -`thinking` SSE lane; see "Extended thinking" below. +Extended thinking is opt-in per node via the `extendedThinking` field and is off by +default. When it is enabled on a reasoning-capable model, the node streams the model's +reasoning over the `thinking` SSE lane on the interactive streaming path only — the +agent / `expectJson` path never requests it; see "Extended thinking" below. --- @@ -37,6 +38,7 @@ extended thinking automatically and streams the model's reasoning over the | `modelSource` | string | Where the model definition comes from (`manual` or `openrouter`) | | `model` | string (custom profile only) | Anthropic model ID, used only when `profile` is `custom` | | `modelTotalTokens` | number (custom profile only) | Total context tokens for the custom profile; must be greater than 0 | +| `extendedThinking` | boolean, `false` | Request extended thinking for this node. Ignored unless the model is reasoning-capable | The model ID and token limits for named profiles are fixed by the profile. Only the `custom` profile exposes `model` and `modelTotalTokens` directly. @@ -61,14 +63,16 @@ Default profile: **Claude Sonnet 4.6**. ## Extended thinking -Whether thinking is requested is driven by the model's `capabilities.reasoning` -flag in the node configuration (stamped from OpenRouter model sync). When set, -the node builds provider-correct thinking parameters based on the model name. +Whether thinking is requested is driven by two things: the model's +`capabilities.reasoning` flag in the node configuration (stamped from OpenRouter +model sync) **and** the node's own `extendedThinking` toggle, which is off by +default. Both must be on. When they are, the node builds provider-correct +thinking parameters based on the model name. Routing prefixes such as `openrouter/anthropic/` are stripped before matching. | Model | Thinking parameters sent | | -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Any Haiku model | None. Haiku has no extended thinking; sending parameters would return a 400. | +| Legacy Claude 3 / 3.5 Haiku | None. Those models have no extended thinking; sending parameters would return a 400. Haiku 4.5 and newer are not excluded and follow the row below. | | `claude-opus-4-7` / `claude-opus-4-8` | `thinking: {type: "adaptive", display: "summarized"}` (adaptive thinking) | | Other Claude models | `thinking: {type: "enabled", budget_tokens: N}` plus the `interleaved-thinking-2025-05-14` beta header, where `N` is half the output-token limit (minimum 2,048, always below `max_tokens`). Skipped entirely if the output window is too small for a valid budget. | diff --git a/nodes/src/nodes/llm_anthropic/anthropic.py b/nodes/src/nodes/llm_anthropic/anthropic.py index 4851c98e2..941fd8fa2 100644 --- a/nodes/src/nodes/llm_anthropic/anthropic.py +++ b/nodes/src/nodes/llm_anthropic/anthropic.py @@ -33,17 +33,10 @@ from ai.common.chat import ChatBase from ai.common.config import Config from ai.common.llm_native_stream import build_anthropic_thinking_kwargs, gate_model_name +from ai.common.utils import parse_bool from langchain_anthropic import ChatAnthropic -# Interim: extended thinking is disabled at the node until the normalized provider adapter -# lands (RFC #1679). With thinking on, Anthropic returns typed content blocks (thinking + text) -# that the agent / expectJson path cannot consume (incident #1658; flatten defense-in-depth in -# #1659). We gate activation here rather than flipping capabilities.reasoning, because that flag -# is stamped by tools/sync_models and re-stamps to true on the next sync. Flip back to re-enable. -_EXTENDED_THINKING_ENABLED = False - - def _estimate_token_ids(text: str) -> list: """Estimate token ids at ~4 chars/token.""" return [0] * max(1, (len(text) + 3) // 4) @@ -78,23 +71,23 @@ def __init__(self, provider: str, connConfig: Dict[str, Any], bag: Dict[str, Any # Init the chat base super().__init__(provider, connConfig, bag) - # Get the LLM - kwargs: Dict[str, Any] = { - 'model': model, - 'api_key': apikey, - 'max_tokens': self._modelOutputTokens, - 'custom_get_token_ids': _estimate_token_ids, - } - if self._is_reasoning and _EXTENDED_THINKING_ENABLED: - kwargs.update(build_anthropic_thinking_kwargs(model_gate, self._modelOutputTokens)) - - self._extended_thinking = bool(kwargs.get('thinking')) - # Only route through the native handler when thinking is actually on; - # non-reasoning models stay on the default LangChain path. + # Extended thinking is opt-in per node (config extendedThinking, default off) and only + # for reasoning models. It is NOT baked into the client: the native streaming adapter adds + # it per call, so thinking activates on the interactive streaming path only — never on the + # agent / expectJson path (which has no streaming and stays on the plain LangChain client). + self._thinking_mode_kwargs: Dict[str, Any] = {} + if self._is_reasoning and parse_bool(config.get('extendedThinking')): + self._thinking_mode_kwargs = build_anthropic_thinking_kwargs(model_gate, self._modelOutputTokens) + self._extended_thinking = bool(self._thinking_mode_kwargs) if self._extended_thinking: self._native_stream_provider = 'anthropic' - self._llm = ChatAnthropic(**kwargs) + self._llm = ChatAnthropic( + model=model, + api_key=apikey, + max_tokens=self._modelOutputTokens, + custom_get_token_ids=_estimate_token_ids, + ) # Save our chat class into the bag bag['chat'] = self diff --git a/nodes/src/nodes/llm_anthropic/services.json b/nodes/src/nodes/llm_anthropic/services.json index 78a338127..9cbdabfc6 100644 --- a/nodes/src/nodes/llm_anthropic/services.json +++ b/nodes/src/nodes/llm_anthropic/services.json @@ -332,6 +332,12 @@ "title": "Tokens", "description": "Total Tokens" }, + "extendedThinking": { + "type": "boolean", + "title": "Extended thinking", + "default": false, + "description": "Enable Anthropic extended thinking (reasoning) for this node. Off by default. Applies to reasoning-capable models on the interactive chat path." + }, "anthropic.custom": { "object": "custom", "properties": [ @@ -345,6 +351,7 @@ "object": "claude-sonnet-4-6", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -352,6 +359,7 @@ "object": "claude-opus-4-6", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -359,6 +367,7 @@ "object": "claude-haiku-4-5", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -366,6 +375,7 @@ "object": "claude-sonnet-4-5", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -373,6 +383,7 @@ "object": "claude-opus-4-5", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -523,6 +534,7 @@ "object": "claude-opus-4-7", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -537,6 +549,7 @@ "object": "claude-fable-5", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -544,6 +557,7 @@ "object": "claude-fable-latest", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -551,6 +565,7 @@ "object": "claude-haiku-latest", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -558,6 +573,7 @@ "object": "claude-opus-4", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -565,6 +581,7 @@ "object": "claude-opus-4-1", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -572,6 +589,7 @@ "object": "claude-opus-4-7-fast", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -579,6 +597,7 @@ "object": "claude-opus-4-8", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -586,6 +605,7 @@ "object": "claude-opus-4-8-fast", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -593,6 +613,7 @@ "object": "claude-opus-latest", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -600,6 +621,7 @@ "object": "claude-sonnet-4", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -607,6 +629,7 @@ "object": "claude-sonnet-5", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -614,6 +637,7 @@ "object": "claude-sonnet-latest", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -621,6 +645,7 @@ "object": "claude-opus-5", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] }, @@ -628,6 +653,7 @@ "object": "claude-opus-5-fast", "properties": [ "llm.cloud.apikey", + "extendedThinking", "llm.cloud.modelSource" ] } diff --git a/nodes/test/agent_crewai/test_stop_words.py b/nodes/test/agent_crewai/test_stop_words.py index 68ee77feb..3f0899c50 100644 --- a/nodes/test/agent_crewai/test_stop_words.py +++ b/nodes/test/agent_crewai/test_stop_words.py @@ -242,7 +242,7 @@ def _load_native_stream(): def _run_native_capture(ns, stop_value): - """Drive _stream_anthropic_messages_api with fakes; return the stop the payload got.""" + """Drive the native Anthropic adapter with fakes; return the stop the payload got.""" captured: dict = {} class _FakeLLM: @@ -258,11 +258,8 @@ class _FakeChat: ns._open_raw_message_stream = lambda client, payload: iter(()) # no events -> no text token = ns.STOP_SEQUENCES_VAR.set(stop_value) try: - # Produces no text, so the function raises after building the payload — we only - # assert the payload wiring, which happens before any streaming. - ns._stream_anthropic_messages_api(_FakeChat(), 'prompt', lambda t: None, None, None) - except RuntimeError: - pass + # Drain the adapter; the payload (with the stop) is built before any streaming. + list(ns.NativeAnthropicAdapter(_FakeChat()).stream('prompt')) finally: ns.STOP_SEQUENCES_VAR.reset(token) return captured.get('stop', 'UNSET') diff --git a/nodes/test/llm_anthropic/test_extended_thinking_init.py b/nodes/test/llm_anthropic/test_extended_thinking_init.py new file mode 100644 index 000000000..781d4810c --- /dev/null +++ b/nodes/test/llm_anthropic/test_extended_thinking_init.py @@ -0,0 +1,100 @@ +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# ============================================================================= + +"""Node-init tests for the llm_anthropic extended-thinking toggle. + +`test_thinking_mode_injected_into_payload_per_call` sets `_thinking_mode_kwargs` +by hand, so it starts after the interesting part. These tests drive the real +chain instead: + + config['extendedThinking'] -> parse_bool -> _is_reasoning gate + -> build_anthropic_thinking_kwargs(model_gate, ...) -> _thinking_mode_kwargs + -> _native_stream_provider = 'anthropic' + +Four ways that chain can break and nothing else would catch: `parse_bool` +reverted to `bool`, the `_is_reasoning` gate dropped, the wrong `model_gate` +passed, or `_native_stream_provider` left unset so the native handler never runs. +""" + +import importlib.util +import os +import sys +import types + +import pytest + +from ai.common.config import Config + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_MOD_PATH = os.path.join(_HERE, '..', '..', 'src', 'nodes', 'llm_anthropic', 'anthropic.py') + +_MODEL = 'claude-sonnet-4-6' + + +def _load_node_module(): + """Load anthropic.py standalone, stubbing the langchain_anthropic client.""" + saved = sys.modules.get('langchain_anthropic') + stub = types.ModuleType('langchain_anthropic') + stub.ChatAnthropic = type('ChatAnthropic', (), {'__init__': lambda self, **kw: None}) + sys.modules['langchain_anthropic'] = stub + try: + spec = importlib.util.spec_from_file_location('_llm_anthropic_node', _MOD_PATH) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + finally: + if saved is None: + sys.modules.pop('langchain_anthropic', None) + else: + sys.modules['langchain_anthropic'] = saved + + +def _build_chat(monkeypatch, *, reasoning: bool, toggle=None): + """Instantiate the node over a controlled config and return it.""" + config = { + 'model': _MODEL, + 'apikey': 'sk-ant-test', + 'modelTotalTokens': 200000, + 'modelOutputTokens': 8192, + 'capabilities': {'reasoning': reasoning}, + } + if toggle is not None: + config['extendedThinking'] = toggle + + # Both the node and ChatBase read their config through this one call. + monkeypatch.setattr(Config, 'getNodeConfig', staticmethod(lambda *a, **k: dict(config))) + return _load_node_module().Chat('anthropic', {}, {}) + + +def test_toggle_absent_leaves_thinking_off(monkeypatch): + chat = _build_chat(monkeypatch, reasoning=True) + + assert chat._thinking_mode_kwargs == {} + assert chat._native_stream_provider == '' + + +def test_toggle_on_reasoning_model_arms_the_native_handler(monkeypatch): + chat = _build_chat(monkeypatch, reasoning=True, toggle=True) + + assert chat._thinking_mode_kwargs != {} + assert chat._native_stream_provider == 'anthropic' + + +def test_toggle_on_non_reasoning_model_stays_off(monkeypatch): + # The capability gate wins: the toggle alone must not request thinking. + chat = _build_chat(monkeypatch, reasoning=False, toggle=True) + + assert chat._thinking_mode_kwargs == {} + assert chat._native_stream_provider == '' + + +@pytest.mark.parametrize('falsy', ['false', 'False', '0', '']) +def test_string_false_from_the_form_stays_off(monkeypatch, falsy): + # The case that fails the moment parse_bool is swapped back for bool(): + # a non-empty string like 'false' is truthy to bool(). + chat = _build_chat(monkeypatch, reasoning=True, toggle=falsy) + + assert chat._thinking_mode_kwargs == {} + assert chat._native_stream_provider == '' diff --git a/packages/ai/src/ai/common/chat.py b/packages/ai/src/ai/common/chat.py index b7da65d77..bcbff3d9e 100644 --- a/packages/ai/src/ai/common/chat.py +++ b/packages/ai/src/ai/common/chat.py @@ -18,9 +18,9 @@ from ai.common.schema import Answer, Question from ai.common.config import Config from ai.common.util import parseJson -from ai.common.utils.content_blocks import flatten_content_blocks from ai.common.validation import validate_model_name, validate_max_tokens, validate_prompt from ai.common.llm_native_stream import STOP_SEQUENCES_VAR, dispatch_native_chat_stream +from ai.common.llm_adapter import LangChainAdapter, NativeOpenAIResponsesAdapter, drive_adapter def _stop_kwargs() -> dict: @@ -39,61 +39,6 @@ def _stop_kwargs() -> dict: return {'stop': stop} if stop else {} -def _make_think_tag_splitter(): - """Split ``...`` CoT out of the content stream (Ollama, Perplexity). - - Returns a ``feed(text) -> (visible, reasoning)`` closure; tags may span deltas. - """ - OPEN, CLOSE = '', '' - state = {'mode': 'visible', 'buf': ''} - - def feed(text: str): - if not text: - return '', '' - buf = state['buf'] + text - visible_parts: list = [] - reasoning_parts: list = [] - while buf: - if state['mode'] == 'visible': - idx = buf.find(OPEN) - if idx < 0: - # Hold back trailing chars that could be a partial ''. - safe = len(buf) - (len(OPEN) - 1) - if safe > 0: - visible_parts.append(buf[:safe]) - buf = buf[safe:] - break - if idx: - visible_parts.append(buf[:idx]) - buf = buf[idx + len(OPEN) :] - state['mode'] = 'thinking' - else: - idx = buf.find(CLOSE) - if idx < 0: - safe = len(buf) - (len(CLOSE) - 1) - if safe > 0: - reasoning_parts.append(buf[:safe]) - buf = buf[safe:] - break - if idx: - reasoning_parts.append(buf[:idx]) - buf = buf[idx + len(CLOSE) :] - state['mode'] = 'visible' - state['buf'] = buf - return ''.join(visible_parts), ''.join(reasoning_parts) - - def flush(): - """Emit anything buffered at end-of-stream (e.g. an unterminated tag).""" - tail = state['buf'] - state['buf'] = '' - if state['mode'] == 'thinking': - return '', tail - return tail, '' - - feed.flush = flush # type: ignore[attr-defined] - return feed - - class ChatBase: """ Abstract base class for all chat drivers with configurable token allocation. @@ -239,15 +184,9 @@ def _chat(self, prompt: str) -> str: Should raise appropriate exceptions for API failures, authentication errors, or other provider-specific issues """ - # Ask the LLM. The stop kwarg is only added when the agent set stop sequences, - # so non-agent callers (and backends/mocks without a stop param) are unaffected. - results = self._llm.invoke(prompt, **_stop_kwargs()) - - # `content` is a plain string on OpenAI-style backends but a list of typed - # blocks on Anthropic with extended thinking. Returning that list verbatim - # hands every caller the repr of the blocks instead of the answer, and the - # reasoning is not the answer in any case. Flatten to the visible text. - text, _reasoning, _sig_only = flatten_content_blocks(results.content) + # Non-streaming: invoke through the adapter — same shared normalization as streaming, + # but a genuinely different mechanism, so the streaming fallback can still recover. + text, _items = LangChainAdapter(self._llm, stream_kwargs=_stop_kwargs()).collect(prompt) return text def getTokens(self, value: str) -> int: @@ -459,70 +398,33 @@ def _chat_string_responses( falling back to non-streaming invoke() only if nothing reached the UI yet. """ prompt = validate_prompt(prompt, self._modelTotalTokens, self.getTokens) - - text_parts: list = [] - finish_reason: Optional[str] = None try: - stream = self._raw_client.responses.create( - model=self._model, - input=prompt, - reasoning={'summary': 'auto'}, - max_output_tokens=self._modelOutputTokens, - stream=True, - ) - for event in stream: - etype = getattr(event, 'type', '') or '' - if etype == 'response.reasoning_summary_text.delta': - delta = getattr(event, 'delta', '') or '' - if delta and on_reasoning_chunk is not None: - on_reasoning_chunk(delta) - elif etype == 'response.output_text.delta': - delta = getattr(event, 'delta', '') or '' - if delta: - text_parts.append(delta) - if on_chunk is not None: - on_chunk(delta) - elif etype == 'response.completed': - resp = getattr(event, 'response', None) - if resp is not None: - status = getattr(resp, 'status', None) - if status == 'completed': - finish_reason = 'stop' - elif status == 'incomplete': - details = getattr(resp, 'incomplete_details', None) - reason = getattr(details, 'reason', None) if details else None - finish_reason = reason or 'length' - else: - finish_reason = status or 'stop' - elif etype in ('response.failed', 'response.error'): - finish_reason = 'error' + adapter = NativeOpenAIResponsesAdapter(self) + text, _items = drive_adapter(adapter, prompt, on_chunk, on_reasoning_chunk) + if not text: + # No text (e.g. response.failed) → route to the fallback below, like the Anthropic path. + raise RuntimeError('OpenAI Responses stream produced no text') + if on_finish is not None: + on_finish(adapter.finish_reason) + return text except Exception as e: warning(f'Reasoning streaming disabled for model={self._model} ({type(e).__name__}): {e}.') - # Only retry non-streaming if nothing has reached the UI; otherwise - # the full fallback would arrive on top of the partial we already streamed. + # Only retry non-streaming if nothing reached the UI; otherwise the full + # fallback would arrive on top of the partial we already streamed. if emitted is None or not emitted['any']: - results = self._llm.invoke(prompt, **_stop_kwargs()) - # str() on a typed-block list yields the blocks' repr, not the - # answer — it would put the model's raw reasoning (and its base64 - # signatures) into the visible output. - content_text, fallback_reasoning, _sig_only = flatten_content_blocks( - getattr(results, 'content', '') or '' - ) - if fallback_reasoning and on_reasoning_chunk is not None: - on_reasoning_chunk(fallback_reasoning) - text_parts = [content_text] - # Push the fallback answer through on_chunk so the open UI bubble - # gets the visible text (the caller dedupes the final pipeline result). + adapter = LangChainAdapter(self._llm, stream_kwargs=_stop_kwargs()) + content_text, _items = adapter.collect(prompt) + # Reasoning reaches its own lane; it must never land in the visible text. + if adapter.reasoning and on_reasoning_chunk is not None: + on_reasoning_chunk(adapter.reasoning) if content_text and on_chunk is not None: on_chunk(content_text) - finish_reason = 'stop' - else: - finish_reason = 'error' - - if on_finish is not None: - on_finish(finish_reason) - - return ''.join(text_parts) + if on_finish is not None: + on_finish('stop') + return content_text + if on_finish is not None: + on_finish('error') + return '' def chat_string( self, @@ -604,50 +506,14 @@ def on_chunk_w(t): result = None if on_chunk_w is not None and _llm is not None and hasattr(_llm, 'stream'): try: - parts = [] - finish_reason: Optional[str] = None - _signature_only_note_sent = False - _think_split = _make_think_tag_splitter() - for piece in _llm.stream(prompt, **_stop_kwargs()): - # content: str for OpenAI-style, list of typed blocks for Anthropic. - content = piece.content - text = '' - thinking_delta = '' - if isinstance(content, list): - # Block vocabulary lives in flatten_content_blocks so the - # non-streaming paths cannot drift away from this one. - text, thinking_delta, sig_only = flatten_content_blocks(content) - if sig_only and not _signature_only_note_sent and on_reasoning_chunk_w is not None: - thinking_delta += ( - '_Extended thinking ran, but this stream only delivered the ' - 'block verification signature, not the readable chain-of-thought ' - 'text. The answer below still reflects internal reasoning._\n\n' - ) - _signature_only_note_sent = True - elif isinstance(content, str): - # Strip inline `...` (Perplexity sonar-reasoning fallback). - text, _thinking_inline = _think_split(content) - if _thinking_inline: - thinking_delta += _thinking_inline - if thinking_delta and on_reasoning_chunk_w is not None: - on_reasoning_chunk_w(thinking_delta) - if text: - on_chunk_w(text) - parts.append(text) - reason = (piece.response_metadata or {}).get('finish_reason') - if reason: - finish_reason = reason - # Drain chars buffered by the splitter (partial-tag tail). - tail_visible, tail_reasoning = _think_split.flush() - if tail_visible: - on_chunk_w(tail_visible) - parts.append(tail_visible) - if tail_reasoning and on_reasoning_chunk_w is not None: - on_reasoning_chunk_w(tail_reasoning) - if parts: - result = ''.join(parts) + # Stream the LangChain path through the normalized adapter; drive_adapter + # fans text/thinking to the callbacks and returns the joined answer. + adapter = LangChainAdapter(_llm, stream_kwargs=_stop_kwargs()) + answer, _items = drive_adapter(adapter, prompt, on_chunk_w, on_reasoning_chunk_w) + if answer: + result = answer if on_finish is not None: - on_finish(finish_reason) + on_finish(adapter.finish_reason) except Exception as e: warning( f'Streaming disabled for model={self._model} ' diff --git a/packages/ai/src/ai/common/llm_adapter.py b/packages/ai/src/ai/common/llm_adapter.py new file mode 100644 index 000000000..5c39b1b00 --- /dev/null +++ b/packages/ai/src/ai/common/llm_adapter.py @@ -0,0 +1,270 @@ +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# ============================================================================= + +"""Normalized LLM provider interface: one Event shape for every provider. + +ChatBase consumes Adapters and never touches provider-native content shapes. +Design: repo discussion #1679 (RFC — virtualized provider Adapter). +""" + +from dataclasses import dataclass, field +from typing import Any, Callable, Iterator, Optional, Protocol, runtime_checkable + +from ai.common.utils import flatten_content_blocks + + +@dataclass +class Event: + """One normalized streaming event: a display delta, or the terminal ``done``.""" + + type: str # "thinking" | "text" | "done" + text: str = '' + items: list[Any] = field(default_factory=list) + + +@runtime_checkable +class Adapter(Protocol): + """Provider adapter: owns provider-native ``history``, streams normalized Events. + + Yields ``Event("thinking"|"text")`` deltas in order, then exactly one terminal + ``Event("done", items=...)``. ``items`` is provider-native and OPAQUE: append it + to ``history`` verbatim — never inspect, edit, reorder, or reserialize it. + """ + + history: list[Any] + + def stream(self, user_text: str) -> Iterator[Event]: ... + + +def drive_adapter( + adapter: Adapter, + user_text: str, + on_text: Optional[Callable[[str], None]] = None, + on_thinking: Optional[Callable[[str], None]] = None, +) -> tuple[str, list[Any]]: + """Consume an adapter's Event stream: fan text/thinking deltas to the sinks, + return the joined answer text and the terminal opaque ``done.items``. + """ + parts: list[str] = [] + items: list[Any] = [] + for ev in adapter.stream(user_text): + if ev.type == 'text': + parts.append(ev.text) + if on_text is not None: + on_text(ev.text) + elif ev.type == 'thinking': + if on_thinking is not None: + on_thinking(ev.text) + elif ev.type == 'done': + items = ev.items + return ''.join(parts), items + + +def _make_think_tag_splitter(): + """Split ``...`` CoT out of the content stream (Ollama, Perplexity). + + Returns a ``feed(text) -> (visible, reasoning)`` closure; tags may span deltas. + """ + OPEN, CLOSE = '', '' + state = {'mode': 'visible', 'buf': ''} + + def feed(text: str): + if not text: + return '', '' + buf = state['buf'] + text + visible_parts: list = [] + reasoning_parts: list = [] + while buf: + if state['mode'] == 'visible': + idx = buf.find(OPEN) + if idx < 0: + # Hold back trailing chars that could be a partial ''. + safe = len(buf) - (len(OPEN) - 1) + if safe > 0: + visible_parts.append(buf[:safe]) + buf = buf[safe:] + break + if idx: + visible_parts.append(buf[:idx]) + buf = buf[idx + len(OPEN) :] + state['mode'] = 'thinking' + else: + idx = buf.find(CLOSE) + if idx < 0: + safe = len(buf) - (len(CLOSE) - 1) + if safe > 0: + reasoning_parts.append(buf[:safe]) + buf = buf[safe:] + break + if idx: + reasoning_parts.append(buf[:idx]) + buf = buf[idx + len(CLOSE) :] + state['mode'] = 'visible' + state['buf'] = buf + return ''.join(visible_parts), ''.join(reasoning_parts) + + def flush(): + """Emit anything buffered at end-of-stream (e.g. an unterminated tag).""" + tail = state['buf'] + state['buf'] = '' + if state['mode'] == 'thinking': + return '', tail + return tail, '' + + feed.flush = flush # type: ignore[attr-defined] + return feed + + +def _make_stream_content_parser(has_reasoning_sink: bool): + """Split streamed content (str, or Anthropic/LangChain-v1 typed blocks) into + (visible_text, reasoning_text); also strips inline ```` from str content. + """ + think_split = _make_think_tag_splitter() + state = {'signature_noted': False} + + def feed(content): + # Inline `` tags are a stateful concern owned by this streaming path; + # the block vocabulary is not, so it stays in flatten_content_blocks. + if isinstance(content, str): + return think_split(content) + text, thinking, sig_only = flatten_content_blocks(content) + if sig_only and not state['signature_noted'] and has_reasoning_sink: + thinking += ( + '_Extended thinking ran, but this stream only delivered the ' + 'block verification signature, not the readable chain-of-thought ' + 'text. The answer below still reflects internal reasoning._\n\n' + ) + state['signature_noted'] = True + return text, thinking + + feed.flush = think_split.flush # type: ignore[attr-defined] + return feed + + +def flatten_content_parts(content: Any) -> tuple[str, str]: + """Collapse a full (non-streamed) message content into (visible_text, reasoning).""" + parse = _make_stream_content_parser(False) + text, thinking = parse(content) + tail, tail_thinking = parse.flush() + return text + tail, thinking + tail_thinking + + +def flatten_content(content: Any) -> str: + """Collapse a full (non-streamed) message content to visible text, dropping thinking. + + Non-streaming callers (agents, expectJson) must get a string, never a block list. + """ + return flatten_content_parts(content)[0] + + +class LangChainAdapter: + """Wraps a LangChain chat model so non-reasoning providers speak the Event contract. + + ``done.items`` is the assistant text turn — LangChain carries no opaque reasoning state. + """ + + def __init__(self, llm: Any, history: list[Any] | None = None, stream_kwargs: dict | None = None): + self.llm = llm + self.history: list[Any] = history if history is not None else [] + self.stream_kwargs = stream_kwargs or {} + self.finish_reason: Optional[str] = None + # Reasoning drained by the last collect(); kept off the visible text. + self.reasoning: str = '' + + def stream(self, user_text: str) -> Iterator[Event]: + self.history.append({'role': 'user', 'content': user_text}) + parse = _make_stream_content_parser(True) + parts: list[str] = [] + for piece in self.llm.stream(self.history, **self.stream_kwargs): + text, thinking = parse(piece.content) + if thinking: + yield Event('thinking', thinking) + if text: + parts.append(text) + yield Event('text', text) + reason = (getattr(piece, 'response_metadata', None) or {}).get('finish_reason') + if reason: + self.finish_reason = reason + tail_text, tail_thinking = parse.flush() + if tail_thinking: + yield Event('thinking', tail_thinking) + if tail_text: + parts.append(tail_text) + yield Event('text', tail_text) + assistant = {'role': 'assistant', 'content': ''.join(parts)} + self.history.append(assistant) + yield Event('done', items=[assistant]) + + def collect(self, user_text: str) -> tuple[str, list[Any]]: + """Non-streaming drain: invoke() + shared normalization. A genuinely different + mechanism from stream(), so it can still recover when streaming fails. + """ + had_history = bool(self.history) + self.history.append({'role': 'user', 'content': user_text}) + # Single-turn callers keep the historical contract: the backend is handed the + # prompt string it was given, not a one-element message list. + payload = self.history if had_history else user_text + result = self.llm.invoke(payload, **self.stream_kwargs) + text, self.reasoning = flatten_content_parts(getattr(result, 'content', '')) + assistant = {'role': 'assistant', 'content': text} + self.history.append(assistant) + return text, [assistant] + + +class NativeOpenAIResponsesAdapter: + """Bridges the OpenAI Responses reasoning stream (create/stream=True) to Events.""" + + def __init__(self, chat: Any): + # Single-turn: history is not accepted (the request is built from user_text, not history). + self.chat = chat + self.history: list[Any] = [] + self.finish_reason: Optional[str] = None + + def stream(self, user_text: str) -> Iterator[Event]: + self.history.append({'role': 'user', 'content': user_text}) + chat = self.chat + parts: list[str] = [] + stream = chat._raw_client.responses.create( + model=chat._model, + input=user_text, + store=False, # stateless: don't retain prompts/responses server-side (30-day default). + reasoning={'summary': 'auto'}, + max_output_tokens=chat._modelOutputTokens, + stream=True, + ) + # try/finally + close() so a raise or early GeneratorExit releases the HTTP + # stream, matching NativeAnthropicAdapter's cleanup. + try: + for event in stream: + etype = getattr(event, 'type', '') or '' + if etype == 'response.reasoning_summary_text.delta': + delta = getattr(event, 'delta', '') or '' + if delta: + yield Event('thinking', delta) + elif etype == 'response.output_text.delta': + delta = getattr(event, 'delta', '') or '' + if delta: + parts.append(delta) + yield Event('text', delta) + elif etype == 'response.completed': + resp = getattr(event, 'response', None) + status = getattr(resp, 'status', None) if resp is not None else None + if status == 'incomplete': + details = getattr(resp, 'incomplete_details', None) + self.finish_reason = (getattr(details, 'reason', None) if details else None) or 'length' + else: + self.finish_reason = 'stop' if status == 'completed' else (status or 'stop') + elif etype in ('response.failed', 'response.error'): + self.finish_reason = 'error' + finally: + closer = getattr(stream, 'close', None) + if callable(closer): + try: + closer() + except Exception: + pass + assistant = {'role': 'assistant', 'content': ''.join(parts)} + self.history.append(assistant) + yield Event('done', items=[assistant]) diff --git a/packages/ai/src/ai/common/llm_native_stream.py b/packages/ai/src/ai/common/llm_native_stream.py index 85c6a3c96..c725aa48b 100644 --- a/packages/ai/src/ai/common/llm_native_stream.py +++ b/packages/ai/src/ai/common/llm_native_stream.py @@ -19,6 +19,8 @@ from rocketlib import debug, warning +from ai.common.llm_adapter import Event, drive_adapter + # Per-call carrier for API-level stop sequences (e.g. CrewAI's ReAct "\nObservation:"). # Set on the ask path in llm_base._question and read at every model sink so the stop # reaches the provider API instead of relying only on post-hoc text truncation. A @@ -56,8 +58,8 @@ def gate_model_name(model: str) -> str: def build_anthropic_thinking_kwargs(model_gate: str, model_output_tokens: int) -> Dict[str, Any]: """Return ``ChatAnthropic`` thinking kwargs by model name, or ``{}`` if unsupported.""" - if 'haiku' in model_gate: - return {} # Haiku has no extended thinking — sending it 400s. + if model_gate.startswith('claude-3') and 'haiku' in model_gate: + return {} # Only legacy Claude 3/3.5 Haiku lack extended thinking; 4.5+ supports it. out: Dict[str, Any] = {} if model_gate.startswith('claude-opus-4-7') or model_gate.startswith('claude-opus-4-8'): out['thinking'] = {'type': 'adaptive', 'display': 'summarized'} @@ -167,67 +169,69 @@ def anthropic_extended_thinking_active(chat: Any) -> bool: return bool(mk.get('thinking')) -def _stream_anthropic_messages_api( - chat: Any, - prompt: str, - on_chunk: Callable[[str], None], - on_finish: Optional[Callable[[Optional[str]], None]], - on_reasoning_chunk: Optional[Callable[[str], None]], -) -> str: - llm = chat._llm - # INVARIANT: read synchronously within the ask() call, while LLMBase._question still - # holds the contextvar (before its finally reset). The stream is consumed here, not - # deferred to the caller, so this never runs after the reset (which would send None). - payload: dict[str, Any] = dict(llm._get_request_payload(prompt, stop=STOP_SEQUENCES_VAR.get() or None, stream=True)) - _raw_client = getattr(llm, '_client', None) - client = _raw_client() if callable(_raw_client) else _raw_client - if client is None: - raise RuntimeError('ChatAnthropic has no _client for native streaming') - - parts: list[str] = [] - finish_reason: Optional[str] = None - reasoning_deltas = 0 - raw_stream = _open_raw_message_stream(client, payload) - - try: - for event in raw_stream: - et = _event_type_name(event) - if et == 'content_block_delta': - delta = getattr(event, 'delta', None) - if delta is None: - continue - dt = _delta_type_name(delta) - if dt == 'thinking_delta' and on_reasoning_chunk is not None: - piece = getattr(delta, 'thinking', None) or '' - if piece: - on_reasoning_chunk(piece) - reasoning_deltas += 1 - elif dt == 'text_delta' and on_chunk is not None: - piece = getattr(delta, 'text', None) or '' - if piece: - on_chunk(piece) - parts.append(piece) - elif et == 'message_delta': - md = getattr(event, 'delta', None) - if md is not None: - sr = getattr(md, 'stop_reason', None) - if sr is not None: - finish_reason = _map_claude_stop_reason(sr) - finally: - closer = getattr(raw_stream, 'close', None) - if callable(closer): - try: - closer() - except Exception: - pass +class NativeAnthropicAdapter: + """Bridges the native Anthropic Messages stream to the Event contract. - if not parts: - raise RuntimeError('Anthropic SDK stream produced no text') + Same payload + raw create(stream=True) path as before; now yields normalized Events. + """ - if on_finish is not None: - on_finish(finish_reason) + def __init__(self, chat: Any): + # Single-turn: history is not accepted (the request is built from user_text, not history). + self.chat = chat + self.history: list = [] + self.finish_reason: Optional[str] = None - return ''.join(parts) + def stream(self, user_text: str): + # INVARIANT: consume synchronously within ask() while STOP_SEQUENCES_VAR is still set. + self.history.append({'role': 'user', 'content': user_text}) + llm = self.chat._llm + payload: dict[str, Any] = dict( + llm._get_request_payload(user_text, stop=STOP_SEQUENCES_VAR.get() or None, stream=True) + ) + # Thinking is added per call (not baked into the client) so it rides only this native + # streaming path — the agent / expectJson path never gets it. + payload.update(getattr(self.chat, '_thinking_mode_kwargs', None) or {}) + _raw_client = getattr(llm, '_client', None) + client = _raw_client() if callable(_raw_client) else _raw_client + if client is None: + raise RuntimeError('ChatAnthropic has no _client for native streaming') + + parts: list[str] = [] + raw_stream = _open_raw_message_stream(client, payload) + try: + for event in raw_stream: + et = _event_type_name(event) + if et == 'content_block_delta': + delta = getattr(event, 'delta', None) + if delta is None: + continue + dt = _delta_type_name(delta) + if dt == 'thinking_delta': + piece = getattr(delta, 'thinking', None) or '' + if piece: + yield Event('thinking', piece) + elif dt == 'text_delta': + piece = getattr(delta, 'text', None) or '' + if piece: + parts.append(piece) + yield Event('text', piece) + elif et == 'message_delta': + md = getattr(event, 'delta', None) + if md is not None: + sr = getattr(md, 'stop_reason', None) + if sr is not None: + self.finish_reason = _map_claude_stop_reason(sr) + finally: + closer = getattr(raw_stream, 'close', None) + if callable(closer): + try: + closer() + except Exception: + pass + + assistant = {'role': 'assistant', 'content': ''.join(parts)} + self.history.append(assistant) + yield Event('done', items=[assistant]) def try_anthropic_native_chat_stream( @@ -242,7 +246,12 @@ def try_anthropic_native_chat_stream( return None try: - text = _stream_anthropic_messages_api(chat, prompt, on_chunk, on_finish, on_reasoning_chunk) + adapter = NativeAnthropicAdapter(chat) + text, _items = drive_adapter(adapter, prompt, on_chunk, on_reasoning_chunk) + if not text: + raise RuntimeError('Anthropic SDK stream produced no text') + if on_finish is not None: + on_finish(adapter.finish_reason) return text except Exception as e: warning( diff --git a/packages/ai/tests/ai/common/test_chat_content.py b/packages/ai/tests/ai/common/test_chat_content.py new file mode 100644 index 000000000..c5f929d2a --- /dev/null +++ b/packages/ai/tests/ai/common/test_chat_content.py @@ -0,0 +1,78 @@ +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# ============================================================================= + +"""Pins _make_stream_content_parser: the provider-shape → (text, reasoning) seam +that the LLM adapters will absorb. Behavior must not drift during that refactor. +""" + +from ai.common.llm_adapter import _make_stream_content_parser, flatten_content + + +def test_str_passthrough(): + # think-splitter holds a possible partial `` tail until flush. + parse = _make_stream_content_parser(True) + text, thinking = parse('hello world') + tail_text, _ = parse.flush() + assert text + tail_text == 'hello world' + assert thinking == '' + + +def test_anthropic_thinking_then_text(): + parse = _make_stream_content_parser(True) + text, thinking = parse( + [ + {'type': 'thinking', 'thinking': 'reasoning...'}, + {'type': 'text', 'text': 'answer'}, + ] + ) + assert text == 'answer' + assert thinking == 'reasoning...' + + +def test_reasoning_block_langchain_v1(): + parse = _make_stream_content_parser(True) + assert parse([{'type': 'reasoning', 'reasoning': 'r'}]) == ('', 'r') + + +def test_text_block_without_type(): + parse = _make_stream_content_parser(True) + assert parse([{'text': 'plain'}]) == ('plain', '') + + +def test_non_dict_blocks_skipped(): + parse = _make_stream_content_parser(True) + assert parse(['stray', {'type': 'text', 'text': 'y'}]) == ('y', '') + + +def test_signature_only_note_emitted_once(): + parse = _make_stream_content_parser(True) + _, first = parse([{'type': 'thinking', 'signature': 'sig1'}]) + assert 'verification signature' in first + _, second = parse([{'type': 'thinking', 'signature': 'sig2'}]) + assert second == '' + + +def test_signature_note_suppressed_without_reasoning_sink(): + parse = _make_stream_content_parser(False) + _, thinking = parse([{'type': 'thinking', 'signature': 'sig'}]) + assert thinking == '' + + +def test_flatten_str_passthrough(): + assert flatten_content('plain answer') == 'plain answer' + + +def test_flatten_anthropic_blocks_drops_thinking(): + # the original crash: a typed-block list must become a plain string. + content = [{'type': 'thinking', 'thinking': 'cot'}, {'type': 'text', 'text': 'answer'}] + assert flatten_content(content) == 'answer' + + +def test_inline_think_split_across_feed_and_flush(): + parse = _make_stream_content_parser(True) + text, thinking = parse('beforecotafter') + tail_text, tail_thinking = parse.flush() + assert text + tail_text == 'beforeafter' + assert thinking + tail_thinking == 'cot' diff --git a/packages/ai/tests/ai/common/test_chat_string_wiring.py b/packages/ai/tests/ai/common/test_chat_string_wiring.py new file mode 100644 index 000000000..c2aae10cb --- /dev/null +++ b/packages/ai/tests/ai/common/test_chat_string_wiring.py @@ -0,0 +1,109 @@ +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# ============================================================================= + +"""Integration: chat_string streams the LangChain path through the normalized adapter.""" + +from ai.common.chat import ChatBase + + +class _Piece: + def __init__(self, content, response_metadata=None): + self.content = content + self.response_metadata = response_metadata + + +class _FakeLLM: + def __init__(self, pieces): + self._pieces = pieces + self.kwargs = None + + def stream(self, messages, **kwargs): + self.kwargs = kwargs + yield from self._pieces + + def invoke(self, messages, **kwargs): + self.kwargs = kwargs + return self._pieces[0] + + +class _Chat(ChatBase): + # Bypass the config-driven __init__; wire only what chat_string reads. + def __init__(self, llm): + self._llm = llm + self._model = 'test-model' + self._modelTotalTokens = 100000 + self._modelOutputTokens = 4096 + self._is_reasoning = False + self._raw_client = None + self._native_stream_provider = '' + self._raw_openai_client = None + + def getTokens(self, value): + return len(value) + + def _ensure_openai_compat_reasoning_stream(self): + pass + + +def test_streams_text_and_reasoning_via_adapter(): + llm = _FakeLLM( + [ + _Piece([{'type': 'thinking', 'thinking': 'cot'}, {'type': 'text', 'text': 'the answer'}]), + ] + ) + chat = _Chat(llm) + chunks, thinks, finishes = [], [], [] + + result = chat.chat_string('hi', on_chunk=chunks.append, on_finish=finishes.append, on_reasoning_chunk=thinks.append) + + assert result == 'the answer' + assert ''.join(chunks) == 'the answer' + assert ''.join(thinks) == 'cot' + + +def test_stop_sequences_reach_the_model(): + from ai.common.llm_native_stream import STOP_SEQUENCES_VAR + + llm = _FakeLLM([_Piece('plain answer here')]) + chat = _Chat(llm) + token = STOP_SEQUENCES_VAR.set(['\nObservation:']) + try: + chat.chat_string('hi', on_chunk=lambda t: None) + finally: + STOP_SEQUENCES_VAR.reset(token) + assert llm.kwargs == {'stop': ['\nObservation:']} + + +def test_chat_nonstreaming_invokes_via_adapter(): + # _chat drains via collect() → invoke() (a different mechanism than stream(), so the + # streaming fallback can recover); content is normalized (thinking dropped) and stop passes. + from ai.common.llm_native_stream import STOP_SEQUENCES_VAR + + llm = _FakeLLM([_Piece([{'type': 'thinking', 'thinking': 'x'}, {'type': 'text', 'text': 'plain answer'}])]) + chat = _Chat(llm) + token = STOP_SEQUENCES_VAR.set(['\nObservation:']) + try: + assert chat._chat('q') == 'plain answer' + finally: + STOP_SEQUENCES_VAR.reset(token) + assert llm.kwargs == {'stop': ['\nObservation:']} + + +def test_responses_no_text_falls_back_to_invoke(): + # response.failed with no text → the no-text guard raises → non-streaming invoke fallback. + class _FailEvent: + type = 'response.failed' + + class _Responses: + def create(self, **kwargs): + return iter([_FailEvent()]) + + class _RawClient: + responses = _Responses() + + chat = _Chat(_FakeLLM([_Piece('fallback answer')])) + chat._raw_client = _RawClient() + result = chat._chat_string_responses('q', on_chunk=lambda t: None, emitted={'any': False}) + assert result == 'fallback answer' diff --git a/packages/ai/tests/ai/common/test_llm_adapter.py b/packages/ai/tests/ai/common/test_llm_adapter.py new file mode 100644 index 000000000..8726f3e0f --- /dev/null +++ b/packages/ai/tests/ai/common/test_llm_adapter.py @@ -0,0 +1,40 @@ +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# ============================================================================= + +"""Pins the normalized Adapter interface: Event shape + structural Adapter check.""" + +from ai.common.llm_adapter import Adapter, Event + + +def test_event_defaults(): + e = Event('text') + assert (e.type, e.text, e.items) == ('text', '', []) + + +def test_event_done_carries_opaque_items(): + e = Event('done', items=[{'role': 'assistant', 'content': 'x'}]) + assert e.type == 'done' + assert e.items == [{'role': 'assistant', 'content': 'x'}] + + +def test_adapter_is_structural_protocol(): + class FakeAdapter: + def __init__(self): + self.history: list = [] + + def stream(self, user_text): + yield Event('thinking', 'reasoning') + yield Event('text', 'answer') + self.history.append('raw-turn') + yield Event('done', items=['raw-turn']) + + fake = FakeAdapter() + assert isinstance(fake, Adapter) + + events = list(fake.stream('question')) + assert events[0] == Event('thinking', 'reasoning') + assert events[1] == Event('text', 'answer') + assert events[-1].type == 'done' and events[-1].items == ['raw-turn'] + assert fake.history == ['raw-turn'] diff --git a/packages/ai/tests/ai/common/test_llm_adapter_drive.py b/packages/ai/tests/ai/common/test_llm_adapter_drive.py new file mode 100644 index 000000000..04fe1c5b3 --- /dev/null +++ b/packages/ai/tests/ai/common/test_llm_adapter_drive.py @@ -0,0 +1,40 @@ +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# ============================================================================= + +"""Pins drive_adapter: Event stream → callbacks + (answer_text, opaque done.items).""" + +from ai.common.llm_adapter import Event, drive_adapter + + +class _FakeAdapter: + def __init__(self, events): + self._events = events + self.history: list = [] + + def stream(self, user_text): + yield from self._events + + +def test_fans_deltas_and_returns_text_and_items(): + events = [ + Event('thinking', 'reasoning'), + Event('text', 'Hel'), + Event('text', 'lo'), + Event('done', items=[{'role': 'assistant', 'content': 'Hello'}]), + ] + texts, thinks = [], [] + answer, items = drive_adapter(_FakeAdapter(events), 'q', texts.append, thinks.append) + + assert answer == 'Hello' + assert items == [{'role': 'assistant', 'content': 'Hello'}] + assert texts == ['Hel', 'lo'] + assert thinks == ['reasoning'] + + +def test_sinks_optional(): + events = [Event('text', 'x'), Event('done', items=[])] + answer, items = drive_adapter(_FakeAdapter(events), 'q') + assert answer == 'x' + assert items == [] diff --git a/packages/ai/tests/ai/common/test_llm_adapter_langchain.py b/packages/ai/tests/ai/common/test_llm_adapter_langchain.py new file mode 100644 index 000000000..f959527b7 --- /dev/null +++ b/packages/ai/tests/ai/common/test_llm_adapter_langchain.py @@ -0,0 +1,102 @@ +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# ============================================================================= + +"""Pins LangChainAdapter: normalizes LangChain content to Events and records history.""" + +from ai.common.llm_adapter import Event, LangChainAdapter + + +class _Piece: + def __init__(self, content, response_metadata=None): + self.content = content + self.response_metadata = response_metadata + + +class _FakeLLM: + def __init__(self, pieces): + self._pieces = pieces + self.seen = None + self.kwargs = None + + def stream(self, messages, **kwargs): + self.seen = list(messages) + self.kwargs = kwargs + for p in self._pieces: + yield p if isinstance(p, _Piece) else _Piece(p) + + def invoke(self, messages, **kwargs): + # Recorded verbatim: collect() hands a str on a single turn, a message list after. + self.seen = messages if isinstance(messages, str) else list(messages) + self.kwargs = kwargs + p = self._pieces[0] + return p if isinstance(p, _Piece) else _Piece(p) + + +def test_normalizes_blocks_and_records_history(): + llm = _FakeLLM([[{'type': 'thinking', 'thinking': 'r'}, {'type': 'text', 'text': 'Hi there'}]]) + adapter = LangChainAdapter(llm) + + events = list(adapter.stream('q')) + + assert Event('thinking', 'r') in events + assert ''.join(e.text for e in events if e.type == 'text') == 'Hi there' + + done = events[-1] + assert done.type == 'done' + assert done.items == [{'role': 'assistant', 'content': 'Hi there'}] + + # history: user turn seen by the model, assistant turn appended after + assert llm.seen == [{'role': 'user', 'content': 'q'}] + assert adapter.history == [ + {'role': 'user', 'content': 'q'}, + {'role': 'assistant', 'content': 'Hi there'}, + ] + + +def test_str_content_flushes_buffered_tail(): + # str content rides the think-splitter, which buffers a possible partial tag. + llm = _FakeLLM(['hello world']) + adapter = LangChainAdapter(llm) + text = ''.join(e.text for e in adapter.stream('q') if e.type == 'text') + assert text == 'hello world' + + +def test_passes_stream_kwargs_and_tracks_finish_reason(): + llm = _FakeLLM([_Piece([{'type': 'text', 'text': 'done'}], response_metadata={'finish_reason': 'stop'})]) + adapter = LangChainAdapter(llm, stream_kwargs={'stop': ['\nObservation:']}) + list(adapter.stream('q')) + assert llm.kwargs == {'stop': ['\nObservation:']} + assert adapter.finish_reason == 'stop' + + +def test_collect_uses_invoke_and_normalizes(): + # collect() is the non-streaming drain: invoke() + shared normalization, thinking dropped. + llm = _FakeLLM([[{'type': 'thinking', 'thinking': 'r'}, {'type': 'text', 'text': 'answer'}]]) + adapter = LangChainAdapter(llm, stream_kwargs={'stop': ['x']}) + + text, items = adapter.collect('q') + + assert text == 'answer' + assert items == [{'role': 'assistant', 'content': 'answer'}] + assert llm.seen == 'q' # single turn: the backend gets the prompt string, as before + assert llm.kwargs == {'stop': ['x']} # stop kwargs threaded to invoke + assert adapter.history == [ + {'role': 'user', 'content': 'q'}, + {'role': 'assistant', 'content': 'answer'}, + ] + + +def test_collect_sends_the_history_once_the_turn_is_not_the_first(): + # Multi-turn: the prior turns are the point, so invoke() gets the message list. + llm = _FakeLLM([[{'type': 'text', 'text': 'second'}]]) + adapter = LangChainAdapter(llm, history=[{'role': 'assistant', 'content': 'first'}]) + + text, _items = adapter.collect('q2') + + assert text == 'second' + assert llm.seen == [ + {'role': 'assistant', 'content': 'first'}, + {'role': 'user', 'content': 'q2'}, # the current turn is included + ] diff --git a/packages/ai/tests/ai/common/test_native_anthropic_adapter.py b/packages/ai/tests/ai/common/test_native_anthropic_adapter.py new file mode 100644 index 000000000..fab0b3c3a --- /dev/null +++ b/packages/ai/tests/ai/common/test_native_anthropic_adapter.py @@ -0,0 +1,122 @@ +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# ============================================================================= + +"""Pins NativeAnthropicAdapter: native Messages stream → Events, same payload path.""" + +from ai.common.llm_adapter import Event +from ai.common.llm_native_stream import ( + NativeAnthropicAdapter, + build_anthropic_thinking_kwargs, + try_anthropic_native_chat_stream, +) + + +class _Delta: + def __init__(self, type, thinking='', text='', stop_reason=None): + self.type = type + self.thinking = thinking + self.text = text + self.stop_reason = stop_reason + + +class _RawEvent: + def __init__(self, type, delta=None): + self.type = type + self.delta = delta + + +class _Messages: + def __init__(self, events): + self._events = events + self.seen = None + + def create(self, **kwargs): + self.seen = kwargs + return iter(self._events) + + +class _Client: + def __init__(self, events): + self.messages = _Messages(events) + + +class _LLM: + def __init__(self, payload, client): + self._payload = payload + self._client = client + + def _get_request_payload(self, prompt, stop=None, stream=False): + return {**self._payload, 'messages': [{'role': 'user', 'content': prompt}]} + + +class _Chat: + def __init__(self, llm, thinking_mode=None): + self._llm = llm + self._extended_thinking = True + self._thinking_mode_kwargs = thinking_mode or {} + + +def _chat_with(events, payload=None): + return _Chat(_LLM(payload or {'model': 'claude-sonnet-4-6', 'max_tokens': 1000}, _Client(events))) + + +def test_native_adapter_yields_events_and_maps_finish(): + events = [ + _RawEvent('content_block_delta', _Delta('thinking_delta', thinking='cot')), + _RawEvent('content_block_delta', _Delta('text_delta', text='answer')), + _RawEvent('message_delta', _Delta('message_delta', stop_reason='end_turn')), + ] + adapter = NativeAnthropicAdapter(_chat_with(events)) + out = list(adapter.stream('q')) + + assert Event('thinking', 'cot') in out + assert Event('text', 'answer') in out + assert out[-1].type == 'done' + assert out[-1].items == [{'role': 'assistant', 'content': 'answer'}] + assert adapter.finish_reason == 'stop' + + +def test_handler_drives_adapter_and_reports_finish(): + events = [ + _RawEvent('content_block_delta', _Delta('text_delta', text='hi')), + _RawEvent('message_delta', _Delta('message_delta', stop_reason='max_tokens')), + ] + finishes = [] + text = try_anthropic_native_chat_stream( + _chat_with(events), 'q', on_chunk=lambda t: None, on_finish=finishes.append, on_reasoning_chunk=lambda t: None + ) + assert text == 'hi' + assert finishes == ['length'] # max_tokens → length + + +def test_thinking_mode_injected_into_payload_per_call(): + client = _Client([_RawEvent('content_block_delta', _Delta('text_delta', text='ok'))]) + chat = _Chat( + _LLM({'model': 'claude-sonnet-4-6', 'max_tokens': 1000}, client), + thinking_mode={'thinking': {'type': 'adaptive', 'display': 'summarized'}}, + ) + list(NativeAnthropicAdapter(chat).stream('q')) + assert client.messages.seen['thinking'] == {'type': 'adaptive', 'display': 'summarized'} + + +def test_no_thinking_in_payload_when_mode_empty(): + client = _Client([_RawEvent('content_block_delta', _Delta('text_delta', text='ok'))]) + chat = _Chat(_LLM({'model': 'm', 'max_tokens': 10}, client)) # no thinking mode + list(NativeAnthropicAdapter(chat).stream('q')) + assert 'thinking' not in client.messages.seen + + +def test_haiku_45_gets_extended_thinking(): + kw = build_anthropic_thinking_kwargs('claude-haiku-4-5', 8192) + assert kw.get('thinking', {}).get('type') == 'enabled' + assert kw['thinking']['budget_tokens'] >= 1024 + + +def test_haiku_latest_gets_extended_thinking(): + assert build_anthropic_thinking_kwargs('claude-haiku-latest', 8192).get('thinking') + + +def test_legacy_claude3_haiku_has_no_thinking(): + assert build_anthropic_thinking_kwargs('claude-3-haiku', 8192) == {} diff --git a/packages/ai/tests/ai/common/test_native_openai_responses_adapter.py b/packages/ai/tests/ai/common/test_native_openai_responses_adapter.py new file mode 100644 index 000000000..42fa1ff55 --- /dev/null +++ b/packages/ai/tests/ai/common/test_native_openai_responses_adapter.py @@ -0,0 +1,136 @@ +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# ============================================================================= + +"""Pins NativeOpenAIResponsesAdapter: Responses stream → Events + finish mapping.""" + +from ai.common.llm_adapter import Event, NativeOpenAIResponsesAdapter + + +class _Ev: + def __init__(self, type, delta='', response=None): + self.type = type + self.delta = delta + self.response = response + + +class _Resp: + def __init__(self, status, incomplete_details=None): + self.status = status + self.incomplete_details = incomplete_details + + +class _Details: + def __init__(self, reason): + self.reason = reason + + +class _Responses: + def __init__(self, events): + self._events = events + self.seen = None + + def create(self, **kwargs): + self.seen = kwargs + return iter(self._events) + + +class _RawClient: + def __init__(self, events): + self.responses = _Responses(events) + + +class _Chat: + def __init__(self, events): + self._raw_client = _RawClient(events) + self._model = 'gpt-5.6' + self._modelOutputTokens = 4096 + + +def test_yields_events_and_completed_maps_to_stop(): + events = [ + _Ev('response.reasoning_summary_text.delta', delta='think'), + _Ev('response.output_text.delta', delta='ans'), + _Ev('response.completed', response=_Resp('completed')), + ] + adapter = NativeOpenAIResponsesAdapter(_Chat(events)) + out = list(adapter.stream('q')) + + assert Event('thinking', 'think') in out + assert Event('text', 'ans') in out + assert out[-1].type == 'done' + assert out[-1].items == [{'role': 'assistant', 'content': 'ans'}] + assert adapter.finish_reason == 'stop' + + +def test_incomplete_maps_to_reason(): + events = [ + _Ev('response.output_text.delta', delta='x'), + _Ev('response.completed', response=_Resp('incomplete', _Details('max_output_tokens'))), + ] + adapter = NativeOpenAIResponsesAdapter(_Chat(events)) + list(adapter.stream('q')) + assert adapter.finish_reason == 'max_output_tokens' + + +def test_failed_response_sets_error_finish(): + adapter = NativeOpenAIResponsesAdapter(_Chat([_Ev('response.failed')])) + list(adapter.stream('q')) + assert adapter.finish_reason == 'error' + + +def test_passes_store_false(): + chat = _Chat([_Ev('response.output_text.delta', delta='x')]) + list(NativeOpenAIResponsesAdapter(chat).stream('q')) + assert chat._raw_client.responses.seen['store'] is False + + +class _ClosableStream: + """Iterator that records close() and can raise mid-iteration.""" + + def __init__(self, events, raise_at=None): + self._events = list(events) + self._i = 0 + self._raise_at = raise_at + self.closed = False + + def __iter__(self): + return self + + def __next__(self): + if self._raise_at is not None and self._i == self._raise_at: + raise RuntimeError('boom') + if self._i >= len(self._events): + raise StopIteration + ev = self._events[self._i] + self._i += 1 + return ev + + def close(self): + self.closed = True + + +class _ChatStream: + def __init__(self, stream): + responses = type('R', (), {'create': lambda _self, **kw: stream})() + self._raw_client = type('C', (), {'responses': responses})() + self._model = 'gpt-5.6' + self._modelOutputTokens = 4096 + + +def test_closes_stream_on_exception(): + stream = _ClosableStream([_Ev('response.output_text.delta', delta='x')], raise_at=1) + try: + list(NativeOpenAIResponsesAdapter(_ChatStream(stream)).stream('q')) + except RuntimeError: + pass + assert stream.closed is True + + +def test_closes_stream_on_early_break(): + stream = _ClosableStream([_Ev('response.output_text.delta', delta=str(i)) for i in range(5)]) + gen = NativeOpenAIResponsesAdapter(_ChatStream(stream)).stream('q') + next(gen) # consume one event, then abandon + gen.close() # GeneratorExit → finally closes the stream + assert stream.closed is True diff --git a/tools/sync_models/src/core/patcher.py b/tools/sync_models/src/core/patcher.py index 5ab34628b..8870cb33b 100644 --- a/tools/sync_models/src/core/patcher.py +++ b/tools/sync_models/src/core/patcher.py @@ -277,7 +277,10 @@ def _detect_namespace(fields: Dict[str, Any]) -> str: return '' -def _repair_field_objects(fields: Dict[str, Any]) -> bool: +_EXTENDED_THINKING = 'extendedThinking' + + +def _repair_field_objects(fields: Dict[str, Any], profiles: Dict[str, Any] | None = None) -> bool: """ Ensure every profile field object that exposes ``llm.cloud.apikey`` also exposes ``llm.cloud.modelSource``, and that ``llm.cloud.modelSource`` is @@ -319,6 +322,27 @@ def _repair_field_objects(fields: Dict[str, Any]) -> bool: break # only one apikey entry per object has_apikey = 'llm.cloud.apikey' in props + + # extendedThinking toggle: present iff the model reasons (capabilities.reasoning). + # Only managed for nodes that DEFINE the field (today: llm_anthropic) so it never + # leaks into providers that neither define nor read it. 'custom' has no capabilities, + # so it is treated as non-reasoning and the toggle is removed (it would be inert). + obj = value.get('object') + if ( + profiles is not None + and _EXTENDED_THINKING in fields + and has_apikey + and isinstance(obj, str) + and obj in profiles + ): + reasons = bool(((profiles.get(obj) or {}).get('capabilities') or {}).get('reasoning')) + if reasons and _EXTENDED_THINKING not in props: + props.append(_EXTENDED_THINKING) + repaired = True + elif not reasons and _EXTENDED_THINKING in props: + props.remove(_EXTENDED_THINKING) + repaired = True + has_model_source = 'llm.cloud.modelSource' in props if has_apikey and not has_model_source: props.append('llm.cloud.modelSource') @@ -357,12 +381,13 @@ def _update_fields_for_added( field_key = f'{namespace}.{profile_key}' - # 1. Field object + # 1. Field object — reasoning models get the extendedThinking toggle (before modelSource). if field_key not in fields: - fields[field_key] = { - 'object': profile_key, - 'properties': ['llm.cloud.apikey', 'llm.cloud.modelSource'], - } + props = ['llm.cloud.apikey', 'llm.cloud.modelSource'] + # Only nodes that define the extendedThinking field get the toggle (today: anthropic). + if _EXTENDED_THINKING in fields and bool((profile.get('capabilities') or {}).get('reasoning')): + props.insert(1, _EXTENDED_THINKING) + fields[field_key] = {'object': profile_key, 'properties': props} # 2 & 3. enum + conditional live inside the profile selector field profile_field = fields.get(f'{namespace}.profile') @@ -518,8 +543,8 @@ def patch( ns = _detect_namespace(fields) - # Repair existing field objects that are missing llm.cloud.modelSource - _repair_field_objects(fields) + # Repair existing field objects (modelSource + capabilities-aware extendedThinking) + _repair_field_objects(fields, updated_profiles) for key in sorted(_added): profile = updated_profiles.get(key, {}) diff --git a/tools/sync_models/src/providers/base.py b/tools/sync_models/src/providers/base.py index c09e829d3..cf4962121 100644 --- a/tools/sync_models/src/providers/base.py +++ b/tools/sync_models/src/providers/base.py @@ -623,7 +623,7 @@ def _run_merge( _fields = json.loads(_raw[_f_start:_f_end]) _fields_copy = copy.deepcopy(_fields) - _needs_repair = _repair_field_objects(_fields_copy) + _needs_repair = _repair_field_objects(_fields_copy, updated_profiles) except Exception: pass diff --git a/tools/sync_models/test/test_extended_thinking_field.py b/tools/sync_models/test/test_extended_thinking_field.py new file mode 100644 index 000000000..ac804f851 --- /dev/null +++ b/tools/sync_models/test/test_extended_thinking_field.py @@ -0,0 +1,82 @@ +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# ============================================================================= + +"""Pins the capabilities-aware `extendedThinking` UI toggle in the field patcher. + +The toggle is only managed for nodes that DEFINE the field (today: llm_anthropic). +Within such a node, reasoning models carry it and non-reasoning models do not, so the +UI stays self-maintaining across model syncs without leaking into other providers. +""" + +from __future__ import annotations + +from core.patcher import _repair_field_objects, _update_fields_for_added + +# A node whose fields define the extendedThinking widget (like llm_anthropic). +_FIELD_DEF = {'extendedThinking': {'type': 'boolean', 'title': 'Extended thinking', 'default': False}} + + +def test_repair_adds_toggle_to_reasoning_and_removes_from_non_reasoning(): + fields = { + **_FIELD_DEF, + 'ns.r': {'object': 'r', 'properties': ['llm.cloud.apikey', 'llm.cloud.modelSource']}, + 'ns.n': {'object': 'n', 'properties': ['llm.cloud.apikey', 'extendedThinking', 'llm.cloud.modelSource']}, + } + profiles = {'r': {'capabilities': {'reasoning': True}}, 'n': {'capabilities': {}}} + + _repair_field_objects(fields, profiles) + + # toggle sits before modelSource (which the patcher keeps last) + assert fields['ns.r']['properties'] == ['llm.cloud.apikey', 'extendedThinking', 'llm.cloud.modelSource'] + assert fields['ns.n']['properties'] == ['llm.cloud.apikey', 'llm.cloud.modelSource'] + + +def test_toggle_never_added_when_field_undefined(): + # A provider node that does NOT define extendedThinking (e.g. openai) must be left alone, + # even for reasoning models — the toggle must not leak in. + fields = {'ns.r': {'object': 'r', 'properties': ['llm.cloud.apikey', 'llm.cloud.modelSource']}} + profiles = {'r': {'capabilities': {'reasoning': True}}} + _repair_field_objects(fields, profiles) + assert fields['ns.r']['properties'] == ['llm.cloud.apikey', 'llm.cloud.modelSource'] + + +def test_repair_removes_inert_toggle_from_custom(): + # 'custom' has no capabilities → non-reasoning → the toggle would be inert, so it is removed. + fields = { + **_FIELD_DEF, + 'ns.custom': { + 'object': 'custom', + 'properties': ['llm.cloud.apikey', 'extendedThinking', 'llm.cloud.modelSource'], + }, + } + _repair_field_objects(fields, {'custom': {}}) + assert fields['ns.custom']['properties'] == ['llm.cloud.apikey', 'llm.cloud.modelSource'] + + +def test_repair_without_profiles_leaves_toggle_untouched(): + fields = { + **_FIELD_DEF, + 'ns.r': {'object': 'r', 'properties': ['llm.cloud.apikey', 'extendedThinking', 'llm.cloud.modelSource']}, + } + _repair_field_objects(fields) # no profiles -> only the modelSource repair runs + assert 'extendedThinking' in fields['ns.r']['properties'] + + +def test_added_reasoning_profile_gets_toggle_when_field_defined(): + fields = {**_FIELD_DEF, 'ns.profile': {'enum': [], 'conditional': []}} + _update_fields_for_added(fields, 'ns', 'newr', {'capabilities': {'reasoning': True}, 'title': 'New R'}, set()) + assert fields['ns.newr']['properties'] == ['llm.cloud.apikey', 'extendedThinking', 'llm.cloud.modelSource'] + + +def test_added_reasoning_profile_no_toggle_when_field_undefined(): + fields = {'ns.profile': {'enum': [], 'conditional': []}} # no extendedThinking field def + _update_fields_for_added(fields, 'ns', 'newr', {'capabilities': {'reasoning': True}, 'title': 'New R'}, set()) + assert fields['ns.newr']['properties'] == ['llm.cloud.apikey', 'llm.cloud.modelSource'] + + +def test_added_non_reasoning_profile_has_no_toggle(): + fields = {**_FIELD_DEF, 'ns.profile': {'enum': [], 'conditional': []}} + _update_fields_for_added(fields, 'ns', 'newn', {'capabilities': {}, 'title': 'New N'}, set()) + assert fields['ns.newn']['properties'] == ['llm.cloud.apikey', 'llm.cloud.modelSource']