From 407d00a6d9223192cddb05d7cfe437fb2a5f3468 Mon Sep 17 00:00:00 2001 From: BILLKISHORE Date: Sun, 2 Aug 2026 15:36:49 +0530 Subject: [PATCH 1/2] style: apply ruff format to docs code blocks Recent ruff releases format Python inside markdown code blocks. These files predate that, so `make check-format` currently fails on an unchanged main. --- README.md | 8 ++- libs/giskard-agents/README.md | 52 +++++++++++------- libs/giskard-checks/README.md | 94 ++++++++++++++++++++------------- libs/giskard-llm/README.md | 24 +++++---- libs/giskard-llm/docs/design.md | 5 +- 5 files changed, 114 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index e95c80d3b7..e5bc1e0236 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ from giskard.checks import Scenario, Groundedness client = OpenAI() + def get_answer(inputs: str) -> str: response = client.chat.completions.create( model="gpt-5-mini", @@ -79,6 +80,7 @@ def get_answer(inputs: str) -> str: ) return response.choices[0].message.content + scenario = ( Scenario("test_dynamic_output") .interact( @@ -121,6 +123,7 @@ Use Giskard Scan to: import asyncio from giskard.scan import vulnerability_scan + async def main(): await vulnerability_scan( target=my_agent, @@ -128,6 +131,7 @@ async def main(): languages=["en"], ) + asyncio.run(main()) ``` @@ -147,11 +151,13 @@ Wrap your model and run the scan: import giskard import pandas as pd + # Replace my_llm_chain with your actual LLM chain or model inference logic def model_predict(df: pd.DataFrame): """The function takes a DataFrame and must return a list of outputs (one per row).""" return [my_llm_chain.run({"query": question}) for question in df["question"]] + giskard_model = giskard.Model( model=model_predict, model_type="text_generation", @@ -183,7 +189,7 @@ knowledge_base = KnowledgeBase.from_pandas(df, columns=["column_1", "column_2"]) testset = generate_testset( knowledge_base, num_questions=60, - language='en', + language="en", agent_description="A customer support chatbot for company X", ) ``` diff --git a/libs/giskard-agents/README.md b/libs/giskard-agents/README.md index 33c53f2a33..6ae31f7e8d 100644 --- a/libs/giskard-agents/README.md +++ b/libs/giskard-agents/README.md @@ -63,8 +63,7 @@ Or add multiple messages to the workflow: ```python # The chat message role is "user" by default. chat = await ( - generator - .chat("You are a helpful assistant.", role="system") + generator.chat("You are a helpful assistant.", role="system") .chat("Hello, how are you?") .chat("I'm fine, thank you!", role="assistant") .chat("What's your name?") @@ -92,7 +91,9 @@ generator = agents.Generator( Or use the convenience method: ```python -generator = agents.Generator(model="openai/gpt-4o-mini").with_retries(5, base_delay=2.0, max_delay=30.0) +generator = agents.Generator(model="openai/gpt-4o-mini").with_retries( + 5, base_delay=2.0, max_delay=30.0 +) ``` ### Rate limiting @@ -109,7 +110,9 @@ generator = agents.Generator( Or use the convenience method: ```python -generator = generator.with_rate_limiter(MinIntervalRateLimiter.from_rpm(60, max_concurrent=5)) +generator = generator.with_rate_limiter( + MinIntervalRateLimiter.from_rpm(60, max_concurrent=5) +) ``` ## Custom middleware @@ -124,6 +127,7 @@ from giskard.agents.generators import GenerationParams from giskard.agents.generators.middleware import CompletionMiddleware, NextFn from giskard.llm.types import ChatMessage, CompletionResponse + @CompletionMiddleware.register("logging") class LoggingMiddleware(CompletionMiddleware): async def call( @@ -138,6 +142,7 @@ class LoggingMiddleware(CompletionMiddleware): logging.info(f"Got response: {response.choices[0].finish_reason}") return response + generator = agents.Generator( model="openai/gpt-4o-mini", middlewares=[LoggingMiddleware()], @@ -154,15 +159,13 @@ each completion call: ```python from pydantic import BaseModel + class SimpleOutput(BaseModel): mood: str greeting: str -chat = await ( - generator.chat("Hello!") - .with_output(SimpleOutput) - .run() -) + +chat = await generator.chat("Hello!").with_output(SimpleOutput).run() assert isinstance(chat.output, SimpleOutput) assert chat.output.mood == "happy" @@ -179,9 +182,7 @@ Here's an example: ```python # This will run a chat with the message "Hello Test Bot, how are you?" chat = await ( - generator.chat( - "Hello {{ name_of_the_bot }}, how are you?", as_template=True - ) + generator.chat("Hello {{ name_of_the_bot }}, how are you?", as_template=True) .with_inputs(name_of_the_bot="Test Bot") .run() ) @@ -246,7 +247,9 @@ You can then load the template as usual: ```python chat = await ( generator.template("evaluators.scientific_theory") - .with_inputs(theory="Normandy is actually the center of the universe because its perfect balance of rain, cheese, and cider creates a quantum field that bends space-time, making it the most harmonious place on Earth.") + .with_inputs( + theory="Normandy is actually the center of the universe because its perfect balance of rain, cheese, and cider creates a quantum field that bends space-time, making it the most harmonious place on Earth." + ) .run() ) @@ -259,10 +262,9 @@ assert score == 5 You can run multiple chats with different inputs by passing a list of inputs to the `run_batch` method. ```python -chats = await ( - generator.chat("What's the weather in {{ city }}?", as_template=True) - .run_batch([{"city": "Paris"}, {"city": "London"}]) -) +chats = await generator.chat( + "What's the weather in {{ city }}?", as_template=True +).run_batch([{"city": "Paris"}, {"city": "London"}]) assert len(chats) == 2 ``` @@ -277,6 +279,7 @@ This can be combined with all functionalities described earlier. Here's an examp ```python from giskard import agents + @agents.tool def get_weather(city: str) -> str: """Get the weather in a city. @@ -291,6 +294,7 @@ def get_weather(city: str) -> str: return f"It's sunny in {city}." + # Run parallel chats with tools chats = await ( generator.chat("Hello, what's the weather in {{ city }}?", as_template=True) @@ -367,10 +371,16 @@ Note: when running a single chat (`workflow.run(...)`), error policy `SKIP` beha from giskard.agents import ErrorPolicy # This may return fewer than 3 chats if some fail. -chats = await generator.chat("Hello!", role="user").on_error(ErrorPolicy.SKIP).run_many(n=3) +chats = ( + await generator.chat("Hello!", role="user").on_error(ErrorPolicy.SKIP).run_many(n=3) +) # This will return 3 chats, some may be in failed state. -chats = await generator.chat("Hello!", role="user").on_error(ErrorPolicy.RETURN).run_many(n=3) +chats = ( + await generator.chat("Hello!", role="user") + .on_error(ErrorPolicy.RETURN) + .run_many(n=3) +) for chat in chats: if chat.failed: @@ -388,8 +398,9 @@ You can change this behavior by passing the `catch=None` on the tool decorator. def get_weather(city: str) -> str: raise ValueError("City not found") + result = await get_weather.run(arguments={"city": "Paris"}) -print(result) # "ERROR: City not found" +print(result) # "ERROR: City not found" # Opt out of the catch @@ -397,6 +408,7 @@ print(result) # "ERROR: City not found" def get_weather(city: str) -> str: raise ValueError("City not found") + # This will raise an exception result = await get_weather.run(arguments={"city": "Paris"}) ``` diff --git a/libs/giskard-checks/README.md b/libs/giskard-checks/README.md index 880581c9aa..1a28a8f6e7 100644 --- a/libs/giskard-checks/README.md +++ b/libs/giskard-checks/README.md @@ -40,7 +40,7 @@ scenario = ( Scenario("test_france_capital") .interact( inputs="What is the capital of France?", - outputs="The capital of France is Paris." + outputs="The capital of France is Paris.", ) .check( Groundedness( @@ -48,7 +48,7 @@ scenario = ( answer_key="trace.last.outputs", context="""France is a country in Western Europe. Its capital and largest city is Paris, known for the Eiffel Tower - and the Louvre Museum.""" + and the Louvre Museum.""", ) ) ) @@ -66,6 +66,7 @@ from giskard.checks import Groundedness, Scenario client = OpenAI() + def get_answer(inputs: str) -> str: response = client.chat.completions.create( model="gpt-5-mini", @@ -73,17 +74,15 @@ def get_answer(inputs: str) -> str: ) return response.choices[0].message.content + scenario = ( Scenario("test_dynamic_output") - .interact( - inputs="What is the capital of France?", - outputs=get_answer - ) + .interact(inputs="What is the capital of France?", outputs=get_answer) .check( Groundedness( name="answer is grounded", answer_key="trace.last.outputs", - context="France is a country in Western Europe..." + context="France is a country in Western Europe...", ) ) ) @@ -94,10 +93,12 @@ The `run()` method is async. In a script, wrap it with `asyncio.run()`: ```python import asyncio + async def main(): result = await scenario.run() print(result) + asyncio.run(main()) ``` @@ -350,18 +351,25 @@ result = await ( Scenario("structured-example") .interact( {"question": "What is the capital of France?"}, - lambda inputs: {"answer": "Paris is the capital of France.", "confidence": 0.95} + lambda inputs: { + "answer": "Paris is the capital of France.", + "confidence": 0.95, + }, + ) + .check( + StringMatching( + name="contains_paris", + keyword="Paris", + text_key="trace.last.outputs.answer", + ) + ) + .check( + Equals( + name="high_confidence", + expected_value=0.95, + key="trace.last.outputs.confidence", + ) ) - .check(StringMatching( - name="contains_paris", - keyword="Paris", - text_key="trace.last.outputs.answer", - )) - .check(Equals( - name="high_confidence", - expected_value=0.95, - key="trace.last.outputs.confidence", - )) .run() ) @@ -381,19 +389,25 @@ result = await ( Scenario("multi_step_conversation") .interact( "Hello, I want to apply for a job.", - lambda inputs: "Hi! I'd be happy to help. Please provide your email." + lambda inputs: "Hi! I'd be happy to help. Please provide your email.", + ) + .check( + LLMJudge( + prompt="The assistant asked for the email politely: {{ trace.last.outputs }}" + ) ) - .check(LLMJudge( - prompt="The assistant asked for the email politely: {{ trace.last.outputs }}" - )) .interact( "My email is test@example.com", - lambda inputs: f"Thank you! I've saved your application with email: {inputs.split()[-1]}" + lambda inputs: ( + f"Thank you! I've saved your application with email: {inputs.split()[-1]}" + ), + ) + .check( + RegexMatching( + pattern="test@example.com", + text_key="trace.last.outputs", + ) ) - .check(RegexMatching( - pattern="test@example.com", - text_key="trace.last.outputs", - )) .run() ) @@ -421,7 +435,7 @@ result = await ( input_generator, lambda inputs: { "response": f"Hi there! Received: {inputs['message']}", - } + }, ) .check(from_fn(lambda trace: True, name="noop")) .run() @@ -469,18 +483,20 @@ result = await ( Scenario("llm-example") .interact( {"question": "What is the capital of France?"}, - lambda inputs: {"answer": "Paris is the capital of France."} + lambda inputs: {"answer": "Paris is the capital of France."}, ) .check(Conformity(name="conformity_check", rule="The agent must answer in English")) - .check(LLMJudge( - name="safety_check", - prompt="""Evaluate if the following response is safe and appropriate. + .check( + LLMJudge( + name="safety_check", + prompt="""Evaluate if the following response is safe and appropriate. Input: {{ trace.last.inputs }} Response: {{ trace.last.outputs }} Return 'passed: true' if safe, 'passed: false' if unsafe.""", - )) + ) + ) .run() ) @@ -544,9 +560,11 @@ For advanced use cases where you need direct control over interactions or trace from giskard.checks import Interaction, TestCase, Trace # Build a Trace manually for a TestCase -trace = Trace(interactions=[ - Interaction(inputs="some text", outputs=process("some text")), -]) +trace = Trace( + interactions=[ + Interaction(inputs="some text", outputs=process("some text")), + ] +) tc = TestCase(trace=trace, checks=[check1, check2], name="advanced_example") test_case_result = await tc.run() ``` @@ -556,8 +574,8 @@ For programmatic test generation or when you need fine-grained control, you can ```python from giskard.checks import ( Scenario, - Interact, # Inherits from `InteractionSpec` - Equals # Inherits from `Check` + Interact, # Inherits from `InteractionSpec` + Equals, # Inherits from `Check` ) scenario = Scenario(name="programmatic_scenario").extend( diff --git a/libs/giskard-llm/README.md b/libs/giskard-llm/README.md index 66fa416784..d75e316309 100644 --- a/libs/giskard-llm/README.md +++ b/libs/giskard-llm/README.md @@ -38,19 +38,25 @@ from giskard.llm import LLMClient client = LLMClient() # Configure with explicit values or env var references -client.configure("openai", api_key="sk-...") # pragma: allowlist secret -client.configure("azure-prod", provider="azure", - api_key="os.environ/AZURE_PROD_KEY", # pragma: allowlist secret +client.configure("openai", api_key="sk-...") # pragma: allowlist secret +client.configure( + "azure-prod", + provider="azure", + api_key="os.environ/AZURE_PROD_KEY", # pragma: allowlist secret base_url="os.environ/AZURE_PROD_ENDPOINT", api_version="2024-02-01", ) -client.configure("anthropic-relaxed", provider="anthropic", - api_key="os.environ/ANTHROPIC_API_KEY", # pragma: allowlist secret +client.configure( + "anthropic-relaxed", + provider="anthropic", + api_key="os.environ/ANTHROPIC_API_KEY", # pragma: allowlist secret merge_system=True, ) response = await client.acompletion("azure-prod/gpt-4o", messages) -response = await client.acompletion("anthropic-relaxed/claude-3-5-haiku-latest", messages) +response = await client.acompletion( + "anthropic-relaxed/claude-3-5-haiku-latest", messages +) ``` ## Provider reference @@ -77,7 +83,7 @@ client = LLMClient() client.configure( "foundry-v1", provider="openai", - api_key="os.environ/AZURE_OPENAI_API_KEY", # pragma: allowlist secret + api_key="os.environ/AZURE_OPENAI_API_KEY", # pragma: allowlist secret base_url="https://example.openai.azure.com/openai/v1/", ) @@ -113,7 +119,7 @@ client = LLMClient() client.configure( "azure-secure", provider="azure_ai", - api_key="os.environ/AZURE_AI_API_KEY", # pragma: allowlist secret + api_key="os.environ/AZURE_AI_API_KEY", # pragma: allowlist secret base_url="os.environ/AZURE_AI_ENDPOINT", http_client=http_client, default_headers={"x-ms-useragent": "giskard-llm"}, @@ -121,7 +127,7 @@ client.configure( client.configure( "google-secure", provider="google", - api_key="os.environ/GEMINI_API_KEY", # pragma: allowlist secret + api_key="os.environ/GEMINI_API_KEY", # pragma: allowlist secret http_client=http_client, ) diff --git a/libs/giskard-llm/docs/design.md b/libs/giskard-llm/docs/design.md index 3d8266ae9d..bb86ad27d5 100644 --- a/libs/giskard-llm/docs/design.md +++ b/libs/giskard-llm/docs/design.md @@ -13,7 +13,10 @@ Input types and output types use different base classes: The Chat Completions API (OpenAI, Azure, Anthropic, Google via `generateContent`) uses a **nested** tool format: ```python -{"type": "function", "function": {"name": "add", "description": "...", "parameters": {...}}} +{ + "type": "function", + "function": {"name": "add", "description": "...", "parameters": {...}}, +} ``` The Responses API (OpenAI) and Interactions API (Google) use a **flat** tool format: From 35e9bfece341a6b86eb9cd2a6d1f2e35e7d89a7e Mon Sep 17 00:00:00 2001 From: BILLKISHORE Date: Sun, 2 Aug 2026 15:38:31 +0530 Subject: [PATCH 2/2] build: pin ruff version in install-tools An unpinned install lets a new ruff release turn `make check` red on an unchanged tree, same reasoning as the existing licensecheck pin. --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 442279cabd..060230e34d 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,13 @@ help: ## Show this help message install: ## Install project dependencies uv sync +# Ruff formatting output changes between releases, so an unpinned install can turn +# `make check` red on a tree nobody touched. Pinned for the same reason as +# LICENSECHECK_VERSION below. +RUFF_VERSION := 0.16.1 + install-tools: ## Install development tools - uv tool install ruff + uv tool install ruff==$(RUFF_VERSION) uv tool install vermin uv tool install basedpyright uv tool install pre-commit --with pre-commit-uv