Skip to content

Commit 53b9aa1

Browse files
authored
fix(prompting): Structured output class invalid chars (#74)
Fix the error when parsing primary types in LLM structured output: ``` "Expected the \'name\' field of a(n) \'json_schema\' \'response_format\' to be at most 64 characters containing a-z, A-Z, 0-9, dashes, or underscores; found: \'TypedResponse[int]\'." ```
1 parent da5699f commit 53b9aa1

2 files changed

Lines changed: 15 additions & 0 deletions

File tree

golden_tests/test_cookbook_examples.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@
6060
"anthropic/claude-sonnet-4-5@20250929",
6161
"deepseek-ai/deepseek-r1-0528",
6262
"deepseek-ai/deepseek-v3.2",
63+
"zai/glm-5",
64+
"google/gemini-3.1-flash-lite-preview",
6365
}
6466

6567
# Models to be used as judges for evaluation.
@@ -406,6 +408,7 @@ def test_dataset_eval(llm, df) -> tuple[float, float]:
406408
"deepseek-ai/deepseek-v3.2",
407409
"qwen/qwen3-235b-a22b-instruct-2507",
408410
"qwen/qwen3-next-80b-a3b-instruct",
411+
"zai/glm-5",
409412
}
410413
)
411414
@kbench.task()
@@ -436,6 +439,7 @@ def test_image_url(llm):
436439
"qwen/qwen3-235b-a22b-instruct-2507",
437440
"qwen/qwen3-next-80b-a3b-instruct",
438441
"anthropic/claude-sonnet-4-5@20250929",
442+
"zai/glm-5",
439443
}
440444
)
441445
@kbench.task()
@@ -466,6 +470,7 @@ def test_image_base64(llm):
466470
"deepseek-ai/deepseek-v3.2",
467471
"qwen/qwen3-235b-a22b-instruct-2507",
468472
"qwen/qwen3-next-80b-a3b-instruct",
473+
"zai/glm-5",
469474
}
470475
)
471476
@kbench.task()
@@ -511,6 +516,7 @@ def run_simple_calculator(a: float, b: float, operator: str) -> float:
511516
"deepseek-ai/deepseek-r1-0528",
512517
"deepseek-ai/deepseek-v3.2",
513518
"google/gemma-3-12b",
519+
"google/gemini-3.1-flash-lite-preview",
514520
}
515521
)
516522
@kbench.task()

src/kaggle_benchmarks/prompting.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,14 @@ def get_payload(self):
6363
class TypedResponse(RenderablePydanticModel, Generic[T]):
6464
value: T
6565

66+
model_config = pydantic.ConfigDict(
67+
title="Response",
68+
extra="forbid",
69+
arbitrary_types_allowed=False,
70+
)
71+
72+
__name__ = "Response"
73+
6674

6775
class ResponseParsingError(ValueError):
6876
"""Error raised when a model response cannot be parsed into the desired schema."""
@@ -178,6 +186,7 @@ def root_model_handler(cls):
178186
@handler(types=(float, int, datetime.datetime, bool))
179187
def primitive_type_handler(cls):
180188
model = TypedResponse[cls]
189+
model.__name__ = "Response"
181190
response = yield (
182191
f"Output JSON using this schema: {json.dumps(model.model_json_schema())}",
183192
model,

0 commit comments

Comments
 (0)