From 7c3958ed05e9c24a067172a41bc629ad126c0ba6 Mon Sep 17 00:00:00 2001 From: rosspeili Date: Fri, 21 Aug 2026 08:18:34 +0300 Subject: [PATCH] Expand test suite with conftest, CLI, core gaps, and example smoke tests. Fixes ARPAHLS/aura#2. Adds shared fixtures, 24 new tests (49 passing), coverage report in CI, and TESTING/CHANGELOG updates. --- .github/workflows/ci.yml | 2 +- .github/workflows/publish-pypi.yml | 2 +- CHANGELOG.md | 7 +- CONTRIBUTING.md | 3 +- docs/TESTING.md | 32 ++++- pyproject.toml | 3 + tests/conftest.py | 68 +++++++++ tests/test_cli.py | 97 +++++++++++++ tests/test_core.py | 11 +- tests/test_core_gaps.py | 216 +++++++++++++++++++++++++++++ tests/test_examples_smoke.py | 19 +++ tests/test_v02.py | 12 +- tests/test_v03.py | 11 +- 13 files changed, 443 insertions(+), 40 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_core_gaps.py create mode 100644 tests/test_examples_smoke.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbe8fba..c0a7fec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,4 +34,4 @@ jobs: - name: flake8 run: flake8 aura tests - name: pytest - run: pytest + run: pytest --cov=aura --cov-report=term-missing diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 0ad658b..08da151 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -35,7 +35,7 @@ jobs: pip install -e ".[dev]" black --check aura tests flake8 aura tests - pytest + pytest --cov=aura --cov-report=term-missing publish: needs: test diff --git a/CHANGELOG.md b/CHANGELOG.md index d27d716..1f67600 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- **PR CI workflow** ([`.github/workflows/ci.yml`](.github/workflows/ci.yml)) — `lint-test` job on pull requests and pushes to `main` (black, flake8, pytest); read-only permissions, no PyPI secrets. +- **Test suite infrastructure** — shared `tests/conftest.py` (`aura_home`, `run_aura` CLI helper), `skillware` pytest marker. +- **`tests/test_cli.py`** — CLI coverage for version, agent CRUD, run, logs, export, export-otel, compare. +- **`tests/test_core_gaps.py`** — config merge, registry archive/legacy ids, hash-chain tamper, compare/export/OTel/runtime/middleware/session modes. +- **`tests/test_examples_smoke.py`** — parametrized smoke runs for all `examples/*/main.py` scripts. +- **CI** — `pytest --cov=aura` report-only output in `lint-test` job. +- **PR CI workflow** ([`.github/workflows/ci.yml`](.github/workflows/ci.yml)) — `lint-test` on pull requests and pushes to `main` (merged via #5). ## [0.3.2] - 2026-08-20 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1350848..acbb640 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -127,7 +127,8 @@ Follow the [Code of Conduct](CODE_OF_CONDUCT.md). We welcome autonomous logical flake8 aura tests ``` -- CI runs on PRs via [`.github/workflows/ci.yml`](.github/workflows/ci.yml) (job **`lint-test`**: pytest, black, flake8). See [TESTING.md](docs/TESTING.md). +- Shared fixtures: **`tests/conftest.py`** (`aura_home`, `run_aura` for CLI subprocess tests). +- CI runs on PRs via [`.github/workflows/ci.yml`](.github/workflows/ci.yml) (job **`lint-test`**: pytest with coverage report, black, flake8). See [TESTING.md](docs/TESTING.md). - Wait for green checks before requesting review. ### CHANGELOG diff --git a/docs/TESTING.md b/docs/TESTING.md index de41e05..064758b 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -12,7 +12,7 @@ pip install -e ".[dev]" ```bash pytest -pytest -v tests/test_v03.py +pytest --cov=aura --cov-report=term-missing ``` ## Lint (required before PR) @@ -46,14 +46,36 @@ pytest **Maintainers:** after the first green `lint-test` run on `main`, enable **branch protection** → required status check **`lint-test`**. +## Coverage expectations + +- **New behavior needs a test** — extend the closest file (`test_core.py`, `test_v02.py`, `test_v03.py`, `test_cli.py`, or `test_core_gaps.py`). +- Shared fixtures live in **`tests/conftest.py`** — do not duplicate `aura_home` in test modules. +- Optional Skillware-only tests use `@pytest.mark.skillware` and `pytest.importorskip("skillware")`. +- CI prints **`pytest --cov=aura --cov-report=term-missing`** for visibility; there is **no coverage gate** yet. + +## Test layout + +| File | Focus | +|---|---| +| `conftest.py` | `aura_home`, `run_aura`, example runner | +| `test_core.py` | Registry, spine, constraints, session export (v0.1) | +| `test_v02.py` | Sequencer, observers, membrane, Skillware host | +| `test_v03.py` | Identity, audit report, hash chain, compare | +| `test_cli.py` | `aura` CLI commands and exit codes | +| `test_core_gaps.py` | Config, exporters, runtime, middleware, archive, tamper | +| `test_examples_smoke.py` | Runnable example scripts | + ## What we test | Area | Tests | |---|---| -| Identity | ULID ids, `agent_ref`, custom `aura_id`, resolve lookup | -| Audit | Hash chain, audit report, approver principal, session export | -| Core | Registry, spine, constraints, conformance, sequencer (see `test_core.py`, `test_v02.py`) | -| Compare / OTel | `test_v03.py` | +| Identity | ULID ids, `agent_ref`, custom `aura_id`, resolve lookup, legacy `AURA-000n`, archive | +| Audit | Hash chain (valid + tamper), audit report, approver principal, session export | +| Core | Registry, spine, constraints, conformance, sequencer (`test_core.py`, `test_v02.py`) | +| CLI | Version, agent CRUD, run, logs, export, export-otel, compare (`test_cli.py`) | +| Config / runtime | YAML merge, `run_script`, middleware, session modes (`test_core_gaps.py`) | +| Compare / OTel | Summary diff, OTel JSONL export (`test_v03.py`, `test_core_gaps.py`) | +| Examples | Smoke run all `examples/*/main.py` (`test_examples_smoke.py`) | ## Pre-PR checklist diff --git a/pyproject.toml b/pyproject.toml index 69a972c..3f101a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,9 @@ include = ["aura*"] [tool.pytest.ini_options] testpaths = ["tests"] pythonpath = ["."] +markers = [ + "skillware: requires skillware extra (pip install -e '.[skillware]')", +] [tool.black] line-length = 100 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..d3ffb5c --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,68 @@ +"""Shared pytest fixtures and helpers.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from collections.abc import Sequence +from pathlib import Path + +import pytest + +from aura import configure + + +@pytest.fixture +def aura_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Isolated AURA_HOME with configure() applied.""" + home = tmp_path / "aura_home" + home.mkdir() + monkeypatch.setenv("AURA_HOME", str(home)) + configure() + return home + + +@pytest.fixture +def project_dir(tmp_path: Path) -> Path: + proj = tmp_path / "project" + proj.mkdir() + return proj + + +@pytest.fixture +def run_aura(aura_home: Path): + """Run `python -m aura.cli.main` with isolated AURA_HOME.""" + + def _run(*args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env["AURA_HOME"] = str(aura_home) + return subprocess.run( + [sys.executable, "-m", "aura.cli.main", *args], + env=env, + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + ) + + return _run + + +def run_example(main_py: Path, aura_home: Path) -> subprocess.CompletedProcess[str]: + """Run an example script with AURA_HOME set.""" + env = os.environ.copy() + env["AURA_HOME"] = str(aura_home) + return subprocess.run( + [sys.executable, str(main_py)], + env=env, + capture_output=True, + text=True, + cwd=str(main_py.parent), + timeout=30, + ) + + +@pytest.fixture +def example_scripts() -> Sequence[Path]: + root = Path(__file__).resolve().parents[1] / "examples" + return sorted(root.glob("*/main.py")) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..6d0a7fc --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,97 @@ +"""CLI integration tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from aura import agent + + +def test_cli_version(run_aura): + result = run_aura("version") + assert result.returncode == 0 + assert "aura-harness" in result.stdout + assert "0.3." in result.stdout + + +def test_cli_agent_create_list_show(run_aura): + create = run_aura( + "agent", + "create", + "demo-bot", + "--ref", + "acme/demo", + "--policy-version", + "2", + ) + assert create.returncode == 0 + data = json.loads(create.stdout) + assert data["agent_ref"] == "acme/demo" + assert data["policy_version"] == "2" + + listing = run_aura("agent", "list") + assert listing.returncode == 0 + assert "acme/demo" in listing.stdout + assert "demo-bot" in listing.stdout + + show = run_aura("agent", "show", "acme/demo") + assert show.returncode == 0 + profile = json.loads(show.stdout) + assert profile["name"] == "demo-bot" + + missing = run_aura("agent", "show", "no-such-agent") + assert missing.returncode == 1 + assert "not found" in missing.stderr + + +def test_cli_run_script(run_aura, tmp_path: Path): + script = tmp_path / "hello.py" + script.write_text("# run under aura session\n", encoding="utf-8") + result = run_aura("run", "cli-runner", str(script)) + assert result.returncode == 0 + payload = json.loads(result.stdout) + assert payload["session_id"].startswith("aura_sess_") + assert "summary" in payload["exports"] + + +def test_cli_logs_export_compare_otel(run_aura, aura_home: Path): + ag = agent("cli-export", agent_ref="test/cli") + with ag.session() as run: + run.emit("turn.start", {}) + run.emit("turn.end", {"tokens": 1}) + session_id = run.session_id + + logs = run_aura("logs", session_id) + assert logs.returncode == 0 + rows = [json.loads(line) for line in logs.stdout.splitlines() if line.strip()] + assert any(r["kind"] == "turn.start" for r in rows) + + export = run_aura("export", session_id) + assert export.returncode == 0 + summary = json.loads(export.stdout) + assert summary["agent_ref"] == "test/cli" + assert summary["audit_report"]["verdict"] == "pass" + + missing = run_aura("export", "missing-session") + assert missing.returncode == 1 + + with ag.session() as run2: + run2.emit("turn.start", {}) + otel = run_aura("export-otel", run2.session_id) + assert otel.returncode == 0 + assert "turn.start" in otel.stdout + otel_path = aura_home / "sessions" / f"{run2.session_id}.otel.jsonl" + assert otel_path.is_file() + + compare = run_aura("compare", session_id, run2.session_id) + assert compare.returncode == 0 + diff = json.loads(compare.stdout) + assert diff["session_a"] == session_id + assert diff["event_count"]["b"] < diff["event_count"]["a"] + + +def test_cli_run_requires_script(run_aura): + result = run_aura("run", "agent-only") + assert result.returncode == 1 + assert "script" in result.stderr.lower() diff --git a/tests/test_core.py b/tests/test_core.py index 207a980..6b1e234 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -7,7 +7,7 @@ import pytest -from aura import agent, configure, ApprovalRequired +from aura import agent, ApprovalRequired from aura.agents.registry import AgentRegistry, DuplicateAgentError from aura.core.constraints import ConstraintEngine, ConstraintContext from aura.core.ids import is_ulid @@ -15,15 +15,6 @@ from aura.core.spectrum import Spectrum -@pytest.fixture -def aura_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - home = tmp_path / "aura_home" - home.mkdir() - monkeypatch.setenv("AURA_HOME", str(home)) - configure() - return home - - def test_spectrum_from_manifest_defaults(): s = Spectrum.from_manifest({}) assert s.level == "mid" diff --git a/tests/test_core_gaps.py b/tests/test_core_gaps.py new file mode 100644 index 0000000..5af2afc --- /dev/null +++ b/tests/test_core_gaps.py @@ -0,0 +1,216 @@ +"""Additional coverage for config, export, runtime, registry, and spine gaps.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from aura import agent +from aura.agents.registry import AgentRegistry +from aura.config import AuraConfig, configure, get_config +from aura.core.compare import compare_sessions, load_summary +from aura.core.spine import AuditSpine, verify_hash_chain, verify_hash_chain_dicts +from aura.exporters.jsonl import export_session +from aura.exporters.otel import export_otel_jsonl, export_session_otel, events_to_spans +from aura.hosts.skillware import SkillwareHost +from aura.runtime.python import aura_wrapped, run_script +from aura.sequencer.middleware import MiddlewarePolicy, MiddlewareStack + + +def test_config_merge_global_and_project(aura_home: Path, project_dir: Path): + (aura_home / "config.yaml").write_text( + yaml.dump({"default_session_mode": "task"}), + encoding="utf-8", + ) + (project_dir / "aura.project.yaml").write_text( + yaml.dump({"storage": "project", "export_on_close": False}), + encoding="utf-8", + ) + cfg = AuraConfig(home=aura_home, project_dir=project_dir) + assert cfg.values["default_session_mode"] == "task" + assert cfg.values["storage"] == "project" + assert cfg.values["export_on_close"] is False + assert cfg.project_aura_dir == project_dir / ".aura" + sessions = cfg.sessions_dir() + assert str(project_dir) in str(sessions) + + +def test_configure_overrides(aura_home: Path): + configure(export_on_close=False) + assert get_config().values["export_on_close"] is False + + +def test_registry_legacy_aura_id_and_archive(aura_home: Path): + reg = AgentRegistry() + legacy = reg.create(name="legacy-bot", aura_id="AURA-0001", agent_ref="legacy/bot") + assert reg.get_by_id("AURA-0001").aura_id == legacy.aura_id + assert reg.resolve("legacy/bot").name == "legacy-bot" + + reg.archive("AURA-0001") + archived = reg.get_by_id("AURA-0001") + assert archived.archived is True + assert reg.list_agents() == [] + assert len(reg.list_agents(include_archived=True)) == 1 + + reg.create(name="legacy-bot", agent_ref="legacy/bot") + assert reg.resolve("legacy/bot").archived is False + + +def test_registry_archive_frees_name_and_ref(aura_home: Path): + reg = AgentRegistry() + first = reg.create(name="reuse", agent_ref="team/reuse") + reg.archive(first.aura_id) + second = reg.create(name="reuse", agent_ref="team/reuse") + assert second.aura_id != first.aura_id + + +def test_hash_chain_detects_tamper(tmp_path: Path): + log = tmp_path / "tampered.jsonl" + spine = AuditSpine("sess-t", "AGENT-1", log_path=log) + spine.append("turn.start", {}) + spine.append("turn.end", {}) + rows = AuditSpine.read_jsonl(log) + rows[1]["content_hash"] = "0" * 64 + assert verify_hash_chain_dicts(rows) is False + assert verify_hash_chain(spine) is True + + +def test_compare_sessions_edge_cases(aura_home: Path): + ag = agent("cmp-edges", policy_version="1") + with ag.session() as run_a: + run_a.emit("turn.start", {}) + with ag.session() as run_b: + run_b.emit("turn.start", {}) + run_b.emit("turn.end", {}) + + path_a = Path(run_a.exports["summary"]) + path_b = Path(run_b.exports["summary"]) + + result = compare_sessions(path_a, path_b) + assert result["conformance"]["same"] is True + assert result["audit_verdict"]["same"] is True + assert result["policy_version"]["a"] == "1" + assert result["event_count"]["delta"] == 1 + + with pytest.raises(FileNotFoundError): + compare_sessions(path_a, path_a.parent / "missing.summary.json") + + +def test_load_summary_roundtrip(aura_home: Path): + ag = agent("summary-load") + with ag.session() as run: + run.emit("turn.start", {}) + path = Path(run.exports["summary"]) + data = load_summary(path) + assert data["session_id"] == run.session_id + + +def test_export_session_writes_summary_and_otel(aura_home: Path): + ag = agent("export-direct", agent_ref="ex/direct", policy_version="3") + with ag.session(export=False) as run: + run.emit("turn.start", {}) + run.emit("turn.end", {"tokens": 2}) + from aura.core.conformance import ConformanceEngine + + conf = ConformanceEngine().summarize(run._session.spine, run._session.rules) + paths = export_session(run._session, aura_home / "sessions", conformance=conf) + assert Path(paths["summary"]).is_file() + assert Path(paths["jsonl"]).is_file() + assert Path(paths["otel"]).is_file() + summary = json.loads(Path(paths["summary"]).read_text(encoding="utf-8")) + assert summary["mode"] == "script" + assert summary["audit_report"]["verdict"] == "pass" + + +def test_otel_export_session_and_spans(aura_home: Path): + ag = agent("otel-gap", policy_version="9") + with ag.session() as run: + run.emit("turn.start", {"note": "x"}) + otel_path = export_session_otel(run.session_id, aura_home / "sessions") + assert otel_path.is_file() + lines = otel_path.read_text(encoding="utf-8").strip().splitlines() + span = json.loads(lines[0]) + assert span["name"] == "membrane.ingress" or span["name"] in { + "membrane.ingress", + "session.open", + "turn.start", + } + events = AuditSpine.read_jsonl(aura_home / "sessions" / f"{run.session_id}.jsonl") + spans = events_to_spans(events) + assert len(spans) == len(events) + export_otel_jsonl(events, aura_home / "sessions" / "manual.otel.jsonl") + assert (aura_home / "sessions" / "manual.otel.jsonl").is_file() + + +def test_session_task_mode_goal(aura_home: Path): + ag = agent("task-mode", default_mode="task") + with ag.session(mode="task") as run: + run.emit("task.start", {"goal": "demo"}) + run.complete_goal({"status": "done"}) + summary = json.loads(Path(run.exports["summary"]).read_text(encoding="utf-8")) + assert summary["mode"] == "task" + kinds = [e.kind for e in run._session.spine.stream()] + assert "task.complete" in kinds + + +def test_session_continuous_mode(aura_home: Path): + ag = agent("continuous-mode", default_mode="continuous") + with ag.session(mode="continuous") as run: + run.emit("turn.start", {}) + summary = json.loads(Path(run.exports["summary"]).read_text(encoding="utf-8")) + assert summary["mode"] == "continuous" + + +def test_runtime_run_script_and_wrapper(aura_home: Path, tmp_path: Path): + script = tmp_path / "sample.py" + script.write_text( + "RESULT = 40 + 2\n", + encoding="utf-8", + ) + handle = agent("runtime-script") + out = run_script(handle, script) + assert out["session_id"].startswith("aura_sess_") + assert "summary" in out["exports"] + + @aura_wrapped(handle) + def add(a: int, b: int) -> int: + return a + b + + assert add(2, 3) == 5 + + +def test_middleware_policy_and_stack(): + assert MiddlewarePolicy.from_manifest({}) is None + policy = MiddlewarePolicy.from_manifest( + {"middleware": {"scope": "per_step", "order": [{"op": "noop", "config": {}}]}} + ) + assert policy is not None + assert policy.scope == "per_step" + stack = MiddlewareStack(policy) + assert stack.run_inbound({"value": 1}) == {"value": 1} + + +def test_skillware_host_register_by_id_and_missing(aura_home: Path): + ag = agent("host-gap") + with ag.session(export=False) as run: + host = SkillwareHost(run._session) + + class _Skill: + skill_id = "wrapped" + + def execute(self, tool: str, args=None): + return {"tool": tool} + + host.register_by_id("wrapped", _Skill()) + assert host.execute("wrapped", "ping", {}) == {"tool": "ping"} + + with pytest.raises(KeyError, match="not registered"): + host.execute("missing", "ping", {}) + + +@pytest.mark.skillware +def test_skillware_extra_import(): + pytest.importorskip("skillware") diff --git a/tests/test_examples_smoke.py b/tests/test_examples_smoke.py new file mode 100644 index 0000000..053671f --- /dev/null +++ b/tests/test_examples_smoke.py @@ -0,0 +1,19 @@ +"""Smoke tests — example scripts run without error.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.conftest import run_example + + +@pytest.mark.parametrize( + "main_py", + sorted((Path(__file__).resolve().parents[1] / "examples").glob("*/main.py")), +) +def test_example_runs(main_py: Path, aura_home): + result = run_example(main_py, aura_home) + assert result.returncode == 0, result.stderr or result.stdout + assert "session" in result.stdout.lower() diff --git a/tests/test_v02.py b/tests/test_v02.py index 1a8d39f..950a029 100644 --- a/tests/test_v02.py +++ b/tests/test_v02.py @@ -4,23 +4,13 @@ import pytest -from aura import agent, configure, ApprovalRequired +from aura import agent, ApprovalRequired from aura.core.conformance import ConformanceEngine from aura.hosts.mock import MockSkill from aura.hosts.skillware import SkillwareHost from aura.observers.base import CallableObserver, get_registry from aura.sequencer import load_steps - -@pytest.fixture -def aura_home(tmp_path, monkeypatch): - home = tmp_path / "aura_home" - home.mkdir() - monkeypatch.setenv("AURA_HOME", str(home)) - configure() - return home - - PIPELINE = { "steps": [ { diff --git a/tests/test_v03.py b/tests/test_v03.py index 1cdc957..f0c4121 100644 --- a/tests/test_v03.py +++ b/tests/test_v03.py @@ -7,7 +7,7 @@ import pytest -from aura import agent, configure, ApprovalRequired +from aura import agent, ApprovalRequired from aura.agents.registry import AgentRegistry, DuplicateAgentError from aura.core.audit_report import AuditReportBuilder from aura.core.compare import compare_sessions @@ -17,15 +17,6 @@ from aura.exporters.otel import events_to_spans -@pytest.fixture -def aura_home(tmp_path, monkeypatch): - home = tmp_path / "aura_home" - home.mkdir() - monkeypatch.setenv("AURA_HOME", str(home)) - configure() - return home - - def test_new_ulid_format(): uid = new_ulid() assert is_ulid(uid)