Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ dev = [
"aiohttp>=3.9.0",
"python-dotenv>=1.0.0",
"nltk",
"renderers>=0.1.8.dev4",
"renderers>=0.1.8.dev28",
]
policy = [
"semgrep>=1.150.0",
Expand All @@ -97,7 +97,7 @@ openenv = [
"openenv-core>=0.3.0",
]
renderers = [
"renderers>=0.1.8.dev4",
"renderers>=0.1.8.dev28",
]
rl = [
"torch>=2.8.0,<2.9.0",
Expand Down
27 changes: 13 additions & 14 deletions tests/test_renderer_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import verifiers as vf
from renderers import RendererPool
from renderers import config_from_name
from renderers.base import ParsedResponse, RenderedTokens, create_renderer
from verifiers.clients.renderer_client import (
RendererClient,
Expand All @@ -24,13 +25,16 @@
)


def test_renderer_client_honors_configured_renderer_name():
def test_renderer_client_honors_configured_renderer_config():
from renderers import Qwen3VLRendererConfig

RendererClient._shared_pools.clear()

cfg = Qwen3VLRendererConfig()
client = object.__new__(RendererClient)
client._renderer = None
client._pool_size = 1
client._config = vf.ClientConfig(client_type="renderer", renderer="qwen3_vl")
client._config = vf.ClientConfig(client_type="renderer", renderer_config=cfg)

sentinel_pool = RendererPool.__new__(RendererPool)
with patch(
Expand All @@ -42,24 +46,23 @@ def test_renderer_client_honors_configured_renderer_name():
assert pool is sentinel_pool
create_pool_mock.assert_called_once_with(
"Qwen/Qwen3-VL-4B-Instruct",
renderer="qwen3_vl",
cfg,
size=1,
tool_parser=None,
reasoning_parser=None,
preserve_all_thinking=False,
preserve_thinking_between_tool_calls=False,
)


def test_renderer_client_uses_renderer_model_name_override():
from renderers import Qwen3VLRendererConfig

RendererClient._shared_pools.clear()

cfg = Qwen3VLRendererConfig()
client = object.__new__(RendererClient)
client._renderer = None
client._pool_size = 1
client._config = vf.ClientConfig(
client_type="renderer",
renderer="qwen3_vl",
renderer_config=cfg,
renderer_model_name="Qwen/Qwen3-VL-4B-Instruct",
)

Expand All @@ -73,12 +76,8 @@ def test_renderer_client_uses_renderer_model_name_override():
assert pool is sentinel_pool
create_pool_mock.assert_called_once_with(
"Qwen/Qwen3-VL-4B-Instruct",
renderer="qwen3_vl",
cfg,
size=1,
tool_parser=None,
reasoning_parser=None,
preserve_all_thinking=False,
preserve_thinking_between_tool_calls=False,
)


Expand Down Expand Up @@ -524,7 +523,7 @@ def _load_tokenizer_and_renderer(model_name: str, renderer_name: str):
from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
renderer = create_renderer(tokenizer, renderer=renderer_name)
renderer = create_renderer(tokenizer, config_from_name(renderer_name))
return tokenizer, renderer


Expand Down
46 changes: 28 additions & 18 deletions tests/test_renderer_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,15 @@
are exercised. Tokenizers come from the local HF cache; no network.
"""

import json
import logging
from typing import Any

import pytest

import verifiers as vf
from datasets import Dataset
from renderers import create_renderer
from renderers import config_from_name, create_renderer
from verifiers.clients.renderer_client import RendererClient, _to_renderer_message
from verifiers.types import Messages, State

Expand Down Expand Up @@ -83,7 +84,7 @@ def _load(model_name: str, renderer_name: str):
from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
renderer = create_renderer(tokenizer, renderer=renderer_name)
renderer = create_renderer(tokenizer, config_from_name(renderer_name))
_renderer_cache[key] = (tokenizer, renderer)
return _renderer_cache[key]

Expand All @@ -106,6 +107,13 @@ def tokenizer_and_renderer(model_family):
# ── Scripted vLLM stand-in ───────────────────────────────────────────


class _ScriptedResponse:
"""httpx.Response stand-in: ``parse_generate_response`` reads ``.content`` as bytes."""

def __init__(self, payload: dict[str, Any]):
self.content = json.dumps(payload).encode()


class ScriptedVLLM:
"""Fake ``AsyncOpenAI``-compatible client serving canned
/inference/v1/generate responses (vllm 0.20 wire shape).
Expand All @@ -124,22 +132,24 @@ async def post(self, path: str, *, cast_to=None, body=None, options=None):
assert self._completions, "ScriptedVLLM ran out of canned completions"
completion_ids = self._completions.pop(0)

return {
"request_id": f"resp-{len(self.requests)}",
"choices": [
{
"index": 0,
"token_ids": list(completion_ids),
"logprobs": {
"content": [
{"token": f"token_id:{tid}", "logprob": -0.1}
for tid in completion_ids
]
},
"finish_reason": "stop",
}
],
}
return _ScriptedResponse(
{
"request_id": f"resp-{len(self.requests)}",
"choices": [
{
"index": 0,
"token_ids": list(completion_ids),
"logprobs": {
"content": [
{"token": f"token_id:{tid}", "logprob": -0.1}
for tid in completion_ids
]
},
"finish_reason": "stop",
}
],
}
)

async def close(self):
pass
Expand Down
Loading
Loading