diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 0000000..5aadc8c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,60 @@ +name: Bug report +description: Report a reproducible AgentRunProof defect +title: "[Bug]: " +labels: + - bug +body: + - type: markdown + attributes: + value: | + Thanks for helping improve AgentRunProof. Never paste API keys, credentials, private traces, or unreviewed production certificates. + - type: input + id: agentrunproof-version + attributes: + label: AgentRunProof version + placeholder: 0.2.0 + validations: + required: true + - type: input + id: sdk-version + attributes: + label: OpenAI Agents SDK version or source commit + description: Use N/A when the defect is limited to installation, CLI parsing, or local certificate checking. + placeholder: openai-agents 0.21.0, source commit abc123, or N/A + - type: input + id: python-version + attributes: + label: Python and operating system + placeholder: Python 3.12.6 on Ubuntu 24.04 x86_64 + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: Minimal reproduction + description: Include the exact command or smallest public-API script. + render: shell + validations: + required: true + - type: textarea + id: observed + attributes: + label: Observed result + description: Include exit status and the relevant invariant/reason, with secrets removed. + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected result + validations: + required: true + - type: checkboxes + id: checks + attributes: + label: Safety check + options: + - label: I removed credentials, private payloads, and unreviewed production data. + required: true + - label: I searched existing issues for the same behavior. + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..cb5136a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Questions and design discussions + url: https://github.com/FU-max-boop/agentrunproof/discussions + about: Ask how to use AgentRunProof or discuss a runtime contract before proposing it. + - name: OpenAI Agents SDK bug + url: https://github.com/openai/openai-agents-python/issues + about: Report SDK behavior directly upstream when it does not depend on AgentRunProof. diff --git a/.github/ISSUE_TEMPLATE/scenario-proposal.yml b/.github/ISSUE_TEMPLATE/scenario-proposal.yml new file mode 100644 index 0000000..8639d5e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/scenario-proposal.yml @@ -0,0 +1,52 @@ +name: Scenario proposal +description: Propose a deterministic runtime contract or regression scenario +title: "[Scenario]: " +labels: + - enhancement +body: + - type: textarea + id: contract + attributes: + label: Runtime contract + description: What public SDK behavior should remain true? + validations: + required: true + - type: textarea + id: counterexample + attributes: + label: Minimal counterexample + description: Describe the smallest public-API run that violates or distinguishes the contract. + validations: + required: true + - type: textarea + id: observations + attributes: + label: Required observations and invariants + description: List only observations that can be recorded deterministically without provider calls. + validations: + required: true + - type: input + id: boundary + attributes: + label: Version or commit boundary + placeholder: 0.20.0 FAIL -> commit abc123 PASS + validations: + required: true + - type: textarea + id: ordinary-test + attributes: + label: Why a normal application test is not enough + description: Explain the cross-version, cross-path, or evidence requirement. + validations: + required: true + - type: checkboxes + id: constraints + attributes: + label: Scenario constraints + options: + - label: The proposed built-in makes no model-provider request and needs no API key. + required: true + - label: The proposal does not contain a benchmark answer, hidden test, credential, or private payload. + required: true + - label: I searched existing scenarios and issues for duplicates. + required: true diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..aadc093 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,19 @@ +## Contract or behavior + +Describe the runtime behavior, evidence contract, or documentation boundary changed. + +## Verification + +List the exact commands and environments used. Include both success and failure-path tests where relevant. + +## Compatibility and evidence + +- [ ] No certificate schema, canonical identifier, supported SDK range, or immutable evidence changes. +- [ ] Or: the compatibility and release impact is explained above. +- [ ] Built-in execution remains provider-free and requires no API key. +- [ ] No credential, private payload, hidden benchmark material, or generated distribution is committed. +- [ ] `pytest`, Ruff, mypy, build, and Twine checks relevant to this change pass. + +## Unverified + +State anything that was not or could not be verified. diff --git a/CHANGELOG.md b/CHANGELOG.md index 9925977..bb8abbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to AgentRunProof will be documented here. +## Unreleased + +- Add a provider-free real-`Runner` tool example, contribution and security guidance, structured + issue and pull-request templates, and a five-minute upstream `RunState` case study. +- Rework the project landing page around a 30-second PyPI check, supported SDK boundaries, honest + upstream impact, and a clear comparison with the SDK's public `ScriptedModel`. +- Improve package discovery metadata and add a reusable social-preview asset. + ## 0.2.0 - 2026-08-15 - Verify released `openai-agents` 0.20.0 and 0.21.0 across Python 3.10–3.14 with an exact diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d22e87c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,62 @@ +# Contributing to AgentRunProof + +AgentRunProof welcomes focused contributions that make OpenAI Agents runtime +behavior easier to reproduce, compare, and verify without a model provider. + +## Good contributions + +- a deterministic scenario for a public SDK runtime contract; +- a minimized regression for streaming, sessions, tools, or resumable state; +- stronger certificate validation or tamper tests; +- compatibility fixes for a released, supported SDK version; +- documentation that makes a result easier to reproduce or interpret. + +Built-in scenarios must be provider-free and deterministic. Do not add API keys, +private user data, hidden benchmark material, or task-specific workarounds. + +## Before writing a scenario + +Open a scenario-proposal issue first when the change introduces a new runtime +contract or certificate field. Describe: + +1. the public SDK behavior being checked; +2. the smallest observable counterexample; +3. the expected invariant and failure reason; +4. the SDK versions or source commits involved; and +5. why an ordinary application test is insufficient. + +Small documentation and clearly isolated bug fixes may go directly to a pull +request. + +## Local setup + +```bash +python -m venv .venv +source .venv/bin/activate +python -m pip install -e ".[test,dev]" +pytest +ruff check . +ruff format --check . +mypy src +``` + +Before submitting, also build and inspect the distributions: + +```bash +python -m build +python -m twine check --strict dist/* +``` + +## Pull requests + +Keep each pull request narrow. Include the failing observation or contract, +tests for both success and failure paths, and the exact commands you ran. State +anything you could not verify. + +Changes to certificate schemas, normalizers, invariant identifiers, canonical +evidence, supported dependency ranges, or publishing workflows require an +explicit compatibility and release-impact note. Never rewrite immutable +historical evidence in place. + +By contributing, you agree that your contribution is licensed under the MIT +License used by this repository. diff --git a/README.md b/README.md index 1850ee9..a8c3c5d 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,75 @@ # AgentRunProof -AgentRunProof is a deterministic runtime-conformance harness for the OpenAI Agents SDK. It drives the real `Runner` with scripted public-`Model` responses, compares observable state across execution paths, and writes a content-addressed conformance record. A failing record carries the normalized counterexample observations. +[![CI](https://github.com/FU-max-boop/agentrunproof/actions/workflows/ci.yml/badge.svg?branch=main&event=push)](https://github.com/FU-max-boop/agentrunproof/actions/workflows/ci.yml?query=branch%3Amain) +[![PyPI](https://img.shields.io/pypi/v/agentrunproof.svg)](https://pypi.org/project/agentrunproof/) +[![Python](https://img.shields.io/pypi/pyversions/agentrunproof.svg)](https://pypi.org/project/agentrunproof/) +[![License: MIT](https://img.shields.io/github/license/FU-max-boop/agentrunproof.svg)](https://github.com/FU-max-boop/agentrunproof/blob/main/LICENSE) + +![AgentRunProof: runtime bugs deserve proofs, not screenshots](https://raw.githubusercontent.com/FU-max-boop/agentrunproof/main/docs/assets/social-preview.png) + +**Catch OpenAI Agents SDK `Runner` regressions without an API key.** + +AgentRunProof runs deterministic scenarios against the real `Runner`, compares observable behavior +across `run` and `run_streamed`, and writes content-addressed JSON records for stream, session, +tool-linkage, and `RunState` resume invariants. A failing record carries the normalized +counterexample observations. AgentRunProof v0.2 declares the `openai-agents>=0.20,<0.22` compatibility window on Python 3.10–3.14. Its packaged CI matrix verifies the exact 0.20.0 and 0.21.0 release baselines. Built-in scenarios make no model API call and require no API key. +> AgentRunProof-backed reports are referenced by two merged maintainer fixes, +> [#4413](https://github.com/openai/openai-agents-python/pull/4413) and +> [#4414](https://github.com/openai/openai-agents-python/pull/4414). This is upstream diagnostic +> impact—not OpenAI adoption, dependency, or endorsement. + +Read the five-minute +[RunState case study](https://github.com/FU-max-boop/agentrunproof/blob/main/docs/case-study-runstate.md) +for the released failure, the recursive follow-up, and the exact before/after evidence chain. + +## 30-second local check + +```bash +python -m pip install "agentrunproof==0.2.0" +agentrunproof probe basic-tool-session-parity --certificate proof.json +agentrunproof check-certificate proof.json +``` + +Expected output: + +```text +PASS basic-tool-session-parity + PASS execution_outcome: OK + PASS stream_parity: OK + PASS tool_linkage: OK + PASS exactly_once: OK + PASS model_script_consumed: OK +certificate_id: sha256:... +written: proof.json +VALID sha256:... PASS +``` + +Exit `0` means PASS, `1` means an observed invariant violation, and `2` means invalid or +unverifiable evidence. See the +[provider-free real Runner example](https://github.com/FU-max-boop/agentrunproof/blob/main/examples/provider_free_tool_demo.py) +and the +[OpenAI Agents integration guide](https://github.com/FU-max-boop/agentrunproof/blob/main/docs/openai-agents.md). + +## Where it fits + +Use the SDK's public `agents.testing.ScriptedModel` with `pytest` for a focused deterministic +application or SDK test. AgentRunProof delegates to `ScriptedModel` on SDK 0.21 and adds reusable +scenario orchestration, automatic `run`/`run_streamed` comparison, multi-phase `RunState` checks, +cross-version evidence, and content-addressed records. + +| You need to… | Start with | +| --- | --- | +| Script model responses and assert one application behavior | `agents.testing.ScriptedModel` + `pytest` | +| Compare the same contract across runner modes or SDK versions | AgentRunProof | +| Check approval/rejection and JSON-restored `RunState` flows | AgentRunProof | +| Share a normalized record that can be checked without a provider call | AgentRunProof | +| Evaluate model-output quality | An eval framework, not AgentRunProof | + ## What AgentRunProof checks - declared completion, interruption, or Runner-exception outcomes for every scenario phase; @@ -134,6 +198,18 @@ isolation defect. The next adoption target is reuse of the recursive regression optional CI check, or a documentation reference—not a default SDK dependency. A community-tool entry was [proposed on the official v0.21 testing-guide PR](https://github.com/openai/openai-agents-python/pull/4381#issuecomment-5293600461). The maintainer [kept that guide limited to SDK-maintained APIs](https://github.com/openai/openai-agents-python/pull/4381#issuecomment-5293704972) while explicitly welcoming future reproducible findings backed by the tool. AgentRunProof therefore remains an external project rather than an official SDK listing or dependency. +## Contribute a runtime contract + +Found a public-API `Runner` inconsistency? +[Open a scenario request](https://github.com/FU-max-boop/agentrunproof/issues/new/choose) with the +exact SDK version and a minimal reproducer. Want to make it permanent? See the +[contribution guide](https://github.com/FU-max-boop/agentrunproof/blob/main/CONTRIBUTING.md) and add +the smallest failing scenario. For usage questions and early contract ideas, use +[Discussions](https://github.com/FU-max-boop/agentrunproof/discussions). + +If AgentRunProof belongs in your regression toolbox, star the repository so other SDK maintainers +can find it. + ## License MIT diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..1086d5c --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,26 @@ +# Security policy + +## Supported versions + +Security fixes are made against the latest released AgentRunProof minor version. +Older evidence remains immutable, but users should reproduce a finding with the +latest release before reporting it when possible. + +## Reporting a vulnerability + +Use GitHub's private vulnerability-reporting form for vulnerabilities in +AgentRunProof itself. Please do not open a public issue for a credential leak, +path escape, certificate-verification bypass, unsafe evidence publication, or +other issue that could put users at risk. + +Include the affected version, a minimal reproduction, impact, and any proposed +mitigation. Remove API keys, production traces, private certificates, and user +data from the report. + +SDK vulnerabilities that do not depend on AgentRunProof should be reported to +the OpenAI Agents SDK maintainers through their +[documented security channel](https://github.com/openai/openai-agents-python/security/policy). + +AgentRunProof executes user-provided Python scenarios and is not a sandbox. A +scenario intentionally running arbitrary code is not itself a vulnerability; +an undocumented privilege boundary bypass or unsafe default may be. diff --git a/docs/assets/social-preview.png b/docs/assets/social-preview.png new file mode 100644 index 0000000..3351213 Binary files /dev/null and b/docs/assets/social-preview.png differ diff --git a/docs/assets/social-preview.svg b/docs/assets/social-preview.svg new file mode 100644 index 0000000..f8ff8ef --- /dev/null +++ b/docs/assets/social-preview.svg @@ -0,0 +1,62 @@ + + AgentRunProof + Deterministic runtime proofs for the OpenAI Agents SDK + + + + + + + + + + + + + + + + + + + + + + + + + AGENTRUNPROOF + Runtime bugs deserve proofs, + not screenshots. + Real Runner · scripted public Model · content-addressed certificate + + + + + + + + + + $ + agentrunproof probe basic-tool-session-parity --certificate proof.json + PASS + execution · stream parity · tool linkage · exactly once + certificate_id: + sha256:6be76236…66fe259 + $ + agentrunproof check-certificate proof.json + VALID PASS + + + + + NO API KEY + + REAL SDK RUNNER + + STREAM + RUNSTATE + community tool · MIT + + + diff --git a/docs/case-study-runstate.md b/docs/case-study-runstate.md new file mode 100644 index 0000000..ee5de47 --- /dev/null +++ b/docs/case-study-runstate.md @@ -0,0 +1,178 @@ +# From one leaked approval to two merged fixes + +## A five-minute AgentRunProof case study + +On 14 August 2026, a small deterministic scenario exposed a state-ownership bug in +`openai-agents==0.20.0`. The report was cited by a maintainer-authored upstream fix, validation of +that fix exposed a deeper recursive boundary, and a second maintainer-authored fix closed the live +and serialized paths. + +The precise result is **maintainer citation plus two merged upstream fixes**. OpenAI has not adopted +AgentRunProof as a dependency, official test tool, or endorsed project. AgentRunProof remains an +independent, community-maintained conformance harness. + +## The bug: two states, one hidden authority + +An interrupted result can be converted into resumable `RunState` objects. Two calls look like two +independent checkpoints: + +```python +paused = await Runner.run(agent, "start") + +decided = paused.to_state() +untouched = paused.to_state() + +decided.approve(decided.get_interruptions()[0]) +resumed = await Runner.run(agent, untouched) +``` + +The expected contract is simple: approving `decided` must not approve `untouched`. The untouched +branch should remain interrupted and the protected tool should execute zero times. + +In the released `openai-agents==0.20.0` wheel, both states retained an alias to the same SDK-owned +approval and invocation ledger through their shared `context_wrapper`. Approving `decided` changed +the authority consulted by `untouched`; resuming the supposedly untouched branch executed the +protected effect once. This was separate from the public interruption-snapshot aliasing being fixed +in upstream [#4409](https://github.com/openai/openai-agents-python/pull/4409). + +The [initial report](https://github.com/openai/openai-agents-python/pull/4409#issuecomment-5291724795) +reduced the behavior to this sibling-state invariant. A maintainer +[confirmed that it would be handled in a follow-up](https://github.com/openai/openai-agents-python/pull/4409#issuecomment-5291774141), +and the content-addressed v0.1.1 +[certificate](https://github.com/FU-max-boop/agentrunproof/blob/e28ef9c8916d02b6d50f84a995401ab337ee0dbb/evidence/current/runstate-sibling-approval-isolation/v1/certificate.json) +and [bundle](https://github.com/FU-max-boop/agentrunproof/blob/e28ef9c8916d02b6d50f84a995401ab337ee0dbb/evidence/current/runstate-sibling-approval-isolation/v1/bundle.json) +made the released failure available without an API key. Their publication is recorded in the +[upstream thread](https://github.com/openai/openai-agents-python/pull/4409#issuecomment-5292251828). + +## Why ordinary tests missed it + +Each happy path was individually plausible. The defect appeared only when the test treated a +resumable result as a branching state machine rather than a single mutable object: + +- A one-checkpoint approval test cannot reveal authority shared by two sibling checkpoints. +- Copying the returned `ToolApprovalItem` does not prove that the internal decision ledger is + detached. +- One `Agent.as_tool()` edge does not exercise recursive ownership. Two or three edges do. +- A live-state fix need not survive `RunState.to_json()` followed by `RunState.from_json()`. +- Checking only completion misses the unsafe outcome: a protected effect can run on the wrong + branch. Its count must be exactly one on the approved branch and zero on the untouched branch. +- `Runner.run()` and `Runner.run_streamed()` are separate execution paths; both need the same state + and side-effect contract. + +These are interaction gaps, not model-quality failures. More provider calls would only add noise. + +## The causal ladder + +The first follow-up, [#4413](https://github.com/openai/openai-agents-python/pull/4413), linked the +original report in its PR body and detached SDK-owned approval state when creating checkpoints. Its +merge commit, [`0b93ce8faa27`](https://github.com/openai/openai-agents-python/commit/0b93ce8faa27d4631df399fe48856b52a8fd9897), +fixed direct sibling isolation. + +Running the wider contract against that exact commit found the next boundary: an approval flattened +through two or more nested `Agent.as_tool()` calls stayed interrupted and never reached the +protected tool. The +[#4413 validation comment](https://github.com/openai/openai-agents-python/pull/4413#issuecomment-5293064242) +reported the direct PASS and recursive FAIL separately. + +The second follow-up, [#4414](https://github.com/openai/openai-agents-python/pull/4414), cited that +recursive finding. Its initial head, +[`9dc7da9f2bd`](https://github.com/openai/openai-agents-python/commit/9dc7da9f2bd7fcf4dc57e57a1f42a8bc2b595c9c), +fixed live recursive routing but still lost both approve and reject decisions after a JSON +round-trip. The revised head, +[`1725a8989ad`](https://github.com/openai/openai-agents-python/commit/1725a8989adcba536641cea1ef56a02d310c534e), +passed live and restored-state checks and was squash-merged as +[`50d65f65c367`](https://github.com/openai/openai-agents-python/commit/50d65f65c367a3b09dcd3313ee8d78471c35885e). +The exact validation scope is recorded in the +[#4414 validation comment](https://github.com/openai/openai-agents-python/pull/4414#issuecomment-5293587925). + +| Exact target | Direct sibling isolation | Recursive live routing | Recursive routing after JSON restoration | +| --- | --- | --- | --- | +| PyPI `openai-agents==0.20.0` | **FAIL**: untouched branch executes the effect | Not claimed by the published comparison | Not claimed by the published comparison | +| `0b93ce8faa27` (#4413 merge) | **PASS** | **FAIL** at two `Agent.as_tool()` edges | Not claimed for this target | +| `9dc7da9f2bd` (#4414 initial head) | Not independently recorded here | **PASS** | **FAIL**: approve and reject remain interrupted | +| `1725a8989ad` / `50d65f65c367` | **PASS** | **PASS** at two and three edges | **PASS**, with decisions applied before or after restoration | + +The canonical v0.2.0 bundle contains five isolated observations for the released wheel, +`0b93ce8faa27`, and `50d65f65c367`. The intermediate `9dc7da9f2bd` result is documented in the +public #4414 validation thread but is not a member of that bundle. + +## What AgentRunProof added + +AgentRunProof turned a surprising outcome into a narrow runtime contract: + +1. **Real Runner:** the scenarios call the SDK's real `Runner.run()` and `Runner.run_streamed()`; + they do not replace the orchestration code under test. +2. **Scripted model:** deterministic responses drive the required tool calls locally. AgentRunProof + delegates to the SDK's public `agents.testing.ScriptedModel` where available and retains a + public-`Model` fallback for SDK 0.20. No model provider or API key is required. +3. **Branch-aware assertions:** one interruption becomes two sibling states; only one receives an + exact approval, while the other must remain pending. +4. **Recursive and durable variants:** the same decision is routed through nested + `Agent.as_tool()` checkpoints, both live and across `RunState.to_json()` / `from_json()`. +5. **Stream parity and exactly-once effects:** non-streaming and terminal-event streaming must agree; + the approved branch executes the synthetic effect once, and the untouched branch executes it + zero times. +6. **Content-addressed evidence:** canonical JSON certificates bind normalized observations. The + comparison bundle binds certificate hashes, clean AgentRunProof source provenance, exact upstream + Git commits, source-built wheel hashes, the released wheel hash, and isolated environment locks. + +That last step matters: a reviewer can distinguish a released-wheel failure from source revisions +that still report package version `0.20.0`. Target identity comes from commit and wheel bytes, not +the version string alone. + +## Reproduce or verify + +Reproduce the released sibling-state counterexample in a fresh environment (exit `1` is the +expected detected invariant violation): + +```bash +python3.12 -m venv .venv-runstate +source .venv-runstate/bin/activate +python -m pip install "agentrunproof==0.2.0" "openai-agents==0.20.0" +mkdir -p build + +agentrunproof probe runstate-sibling-approval-isolation \ + --certificate build/runstate-sibling.json || test "$?" -eq 1 +agentrunproof check-certificate build/runstate-sibling.json +``` + +Independently validate the published five-run before/after record without rerunning the SDK: + +```bash +git clone --branch v0.2.0 --depth 1 \ + https://github.com/FU-max-boop/agentrunproof.git +cd agentrunproof +python3.12 -m venv .venv +source .venv/bin/activate +python -m pip install . +agentrunproof check-upstream-bundle \ + evidence/upstream-comparison/v2/bundle.json +``` + +The published [AgentRunProof v0.2.0 release](https://github.com/FU-max-boop/agentrunproof/releases/tag/v0.2.0) +and [PyPI package](https://pypi.org/project/agentrunproof/0.2.0/) correspond to source tag `v0.2.0`, +which points to +[`a2744b06e6cb`](https://github.com/FU-max-boop/agentrunproof/commit/a2744b06e6cb977e723e4d9f24eccdda59c3c7a5). +Its [comparison bundle](https://github.com/FU-max-boop/agentrunproof/blob/a2744b06e6cb977e723e4d9f24eccdda59c3c7a5/evidence/upstream-comparison/v2/bundle.json) +has ID `sha256:da23130794081eaaaa0321953e596a62c5675a451711f552a1f31abb3f5a6349` +and was generated from clean harness source +[`d94443b5ef1f`](https://github.com/FU-max-boop/agentrunproof/commit/d94443b5ef1f832ce8e3673fb29f59fb67044a89). +The exact Linux x86_64 / CPython 3.12.13 regeneration completed in the public +[canonical CI job](https://github.com/FU-max-boop/agentrunproof/actions/runs/31874605394/job/94988518881), +which rebuilt the wheels, reran all five observations, and required byte-identical bundle members. + +## Limits and recognition boundary + +This evidence covers only the named `RunState` approval/resume scenarios and exact targets above. +It uses synthetic deterministic effects and terminal streaming events; it does not evaluate model +quality, provider behavior, token timing, backpressure, cancellation, Realtime, Voice, or general +SDK correctness. The socket-deny guard covers isolated scenario execution, not artifact download or +wheel building. + +Content addressing detects modification and binds provenance inside the bundle, but it is not a +signature, publisher authentication, or proof that an untrusted machine executed the stated +command. Local bundle checking validates the record; it does not rerun the SDK. + +Most importantly, the public upstream record supports this claim: **OpenAI Agents maintainers cited +the findings, authored the fixes, and merged #4413 and #4414.** It does not support the stronger +claim that OpenAI adopted, certified, endorsed, or depends on AgentRunProof. diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..a84318e --- /dev/null +++ b/examples/README.md @@ -0,0 +1,44 @@ +# Provider-free real Runner demo + +This example drives the real OpenAI Agents SDK `Runner` through a two-turn +function-tool loop using AgentRunProof's `DeterministicModel`. It makes exactly +one local tool call, consumes the complete model script, and makes zero model +provider requests. + +After installing AgentRunProof, run either SDK execution path from the repository +root. No API key is required: + +```bash +python examples/provider_free_tool_demo.py +python examples/provider_free_tool_demo.py --mode streamed +``` + +The command prints a machine-readable summary like this: + +```json +{ + "final_output": "The fixture value is 42.", + "mode": "run", + "model_calls": 2, + "provider_requests": 0, + "script_consumed": true, + "tool": { + "arguments": [ + "alpha" + ], + "invocations": 1, + "name": "lookup_fixture" + } +} +``` + +Tracing is explicitly disabled so the demo stays local even if the surrounding +environment has an SDK trace exporter configured. The `run` and `streamed` +variants use the same deterministic script and assert the same final output and +exactly-once side effect. + +To run the example's tests: + +```bash +pytest -q examples/test_provider_free_tool_demo.py +``` diff --git a/examples/provider_free_tool_demo.py b/examples/provider_free_tool_demo.py new file mode 100644 index 0000000..fdc03d4 --- /dev/null +++ b/examples/provider_free_tool_demo.py @@ -0,0 +1,133 @@ +"""Run a real Agents SDK tool loop without a model provider or API key.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +from dataclasses import dataclass +from typing import Literal + +from agents import Agent, Runner, function_tool +from agents.run import RunConfig + +from agentrunproof import DeterministicModel, assistant_message, function_call + +RunMode = Literal["run", "streamed"] + + +@dataclass(frozen=True) +class DemoResult: + """The small set of facts the demo proves about the completed run.""" + + mode: RunMode + final_output: str + tool_invocations: tuple[str, ...] + model_calls: int + + def as_json(self) -> dict[str, object]: + return { + "final_output": self.final_output, + "mode": self.mode, + "model_calls": self.model_calls, + "provider_requests": 0, + "script_consumed": True, + "tool": { + "arguments": list(self.tool_invocations), + "invocations": len(self.tool_invocations), + "name": "lookup_fixture", + }, + } + + +async def run_demo(mode: RunMode = "run") -> DemoResult: + """Drive one deterministic function-tool call through the real SDK Runner.""" + + tool_invocations: list[str] = [] + + @function_tool + def lookup_fixture(key: str) -> str: + """Return the value for one public, synthetic fixture key.""" + + tool_invocations.append(key) + return "42" if key == "alpha" else "not-found" + + model = DeterministicModel( + [ + [ + function_call( + "lookup_fixture", + {"key": "alpha"}, + call_id="provider-free-demo-call", + ) + ], + [assistant_message("The fixture value is 42.")], + ] + ) + agent = Agent( + name="Provider-free demo", + instructions="Look up the requested fixture, then report its value.", + model=model, + tools=[lookup_fixture], + ) + run_config = RunConfig(tracing_disabled=True) + + if mode == "streamed": + streamed_result = Runner.run_streamed( + agent, + "Look up the synthetic fixture named alpha.", + run_config=run_config, + ) + async for _ in streamed_result.stream_events(): + pass + final_output = streamed_result.final_output + else: + run_result = await Runner.run( + agent, + "Look up the synthetic fixture named alpha.", + run_config=run_config, + ) + final_output = run_result.final_output + + model.assert_complete() + if final_output != "The fixture value is 42.": + raise AssertionError(f"Unexpected final output: {final_output!r}") + if tool_invocations != ["alpha"]: + raise AssertionError( + f"Expected exactly one lookup_fixture('alpha') call, got {tool_invocations!r}" + ) + if len(model.calls) != 2: + raise AssertionError(f"Expected two model turns, got {len(model.calls)}") + expected_streamed = mode == "streamed" + if any(call.streamed is not expected_streamed for call in model.calls): + raise AssertionError(f"The model was not called consistently in {mode!r} mode") + + return DemoResult( + mode=mode, + final_output=final_output, + tool_invocations=tuple(tool_invocations), + model_calls=len(model.calls), + ) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run a real OpenAI Agents SDK tool loop with no provider request." + ) + parser.add_argument( + "--mode", + choices=("run", "streamed"), + default="run", + help="Use Runner.run (default) or Runner.run_streamed.", + ) + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + result = asyncio.run(run_demo(args.mode)) + print(json.dumps(result.as_json(), indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/examples/test_provider_free_tool_demo.py b/examples/test_provider_free_tool_demo.py new file mode 100644 index 0000000..a401a39 --- /dev/null +++ b/examples/test_provider_free_tool_demo.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import asyncio +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from provider_free_tool_demo import run_demo + + +@pytest.mark.parametrize("mode", ["run", "streamed"]) +def test_real_runner_executes_local_tool_exactly_once(mode: str) -> None: + result = asyncio.run(run_demo(mode)) + + assert result.mode == mode + assert result.final_output == "The fixture value is 42." + assert result.tool_invocations == ("alpha",) + assert result.model_calls == 2 + + +@pytest.mark.parametrize("mode", ["run", "streamed"]) +def test_demo_runs_directly_without_an_api_key(mode: str) -> None: + examples_dir = Path(__file__).resolve().parent + environment = os.environ.copy() + environment.pop("OPENAI_API_KEY", None) + completed = subprocess.run( + [sys.executable, str(examples_dir / "provider_free_tool_demo.py"), "--mode", mode], + cwd=examples_dir.parent, + env=environment, + check=True, + capture_output=True, + text=True, + ) + + summary = json.loads(completed.stdout) + assert summary == { + "final_output": "The fixture value is 42.", + "mode": mode, + "model_calls": 2, + "provider_requests": 0, + "script_consumed": True, + "tool": { + "arguments": ["alpha"], + "invocations": 1, + "name": "lookup_fixture", + }, + } diff --git a/pyproject.toml b/pyproject.toml index d2b4aa3..26e87a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,11 +5,18 @@ build-backend = "hatchling.build" [project] name = "agentrunproof" dynamic = ["version"] -description = "Deterministic runtime conformance for the OpenAI Agents SDK" +description = "Provider-free Runner regression probes for the OpenAI Agents SDK" readme = "README.md" requires-python = ">=3.10" license = { file = "LICENSE" } authors = [{ name = "Fu Xiaonan" }] +keywords = [ + "agent-testing", + "conformance-testing", + "openai-agents", + "regression-testing", + "runstate", +] dependencies = [ "openai-agents>=0.20.0,<0.22", ] @@ -26,6 +33,7 @@ classifiers = [ "Programming Language :: Python :: 3.14", "Topic :: Software Development :: Testing", "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Typing :: Typed", ] [project.urls] @@ -35,6 +43,8 @@ Issues = "https://github.com/FU-max-boop/agentrunproof/issues" Changelog = "https://github.com/FU-max-boop/agentrunproof/blob/main/CHANGELOG.md" Documentation = "https://github.com/FU-max-boop/agentrunproof/blob/main/docs/openai-agents.md" Evidence = "https://github.com/FU-max-boop/agentrunproof/tree/main/evidence" +CaseStudy = "https://github.com/FU-max-boop/agentrunproof/blob/main/docs/case-study-runstate.md" +Community = "https://github.com/FU-max-boop/agentrunproof/discussions" [project.scripts] agentrunproof = "agentrunproof.cli:main" @@ -58,9 +68,12 @@ packages = ["src/agentrunproof"] [tool.hatch.build.targets.sdist] include = [ "/CHANGELOG.md", - "/docs/openai-agents.md", + "/CONTRIBUTING.md", + "/docs/**", + "/examples/**", "/LICENSE", "/README.md", + "/SECURITY.md", "/pyproject.toml", "/requirements/history/**", "/requirements/upstream/**", @@ -74,6 +87,7 @@ exclude = [ "/.gitignore", "/build/**", "/dist/**", + "/docs/assets/social-preview.png", "/evidence/**", ] @@ -83,7 +97,7 @@ path = "src/agentrunproof/_version.py" [tool.pytest.ini_options] addopts = "-ra" asyncio_mode = "auto" -testpaths = ["tests"] +testpaths = ["tests", "examples"] [tool.ruff] line-length = 100