test(verify): language-neutral verification conformance vectors

python/tests/test_vectors.py

@@ -0,0 +1,76 @@
+"""Conformance: the reference engine must reproduce every language-neutral vector.
+
+The vectors in ``tests/vectors/`` are a portable contract (see
+``tests/vectors/README.md``) intended to be consumed by SDKs in any language.
+This test guards the Python reference implementation against them: for each
+vector it loads the manifest + context, runs :func:`verify_manifest`, and
+asserts the expected overall result and per-field statuses.
+
+Conformance IDs: AM-VEC-001 .. AM-VEC-015.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from agent_manifest._verify import (
+    RevocationRecord,
+    RevocationStore,
+    VerificationContext,
+    verify_manifest,
+)
+
+VECTORS_DIR = Path(__file__).parent / "vectors"
+
+
+def _load_index() -> list[dict[str, Any]]:
+    index = json.loads((VECTORS_DIR / "index.json").read_text())
+    return index["vectors"]
+
+
+def _load_vector(file_name: str) -> dict[str, Any]:
+    return json.loads((VECTORS_DIR / file_name).read_text())
+
+
+VECTOR_FILES = [entry["file"] for entry in _load_index()]
+
+
+def test_index_lists_every_vector_file() -> None:
+    on_disk = {p.name for p in VECTORS_DIR.glob("AM-VEC-*.json")}
+    in_index = set(VECTOR_FILES)
+    assert on_disk == in_index, "index.json is out of sync with the vector files"
+
+
+@pytest.mark.parametrize("file_name", VECTOR_FILES, ids=[f.removesuffix(".json") for f in VECTOR_FILES])
+def test_vector(file_name: str) -> None:
+    vector = _load_vector(file_name)
+
+    store = RevocationStore()
+    if vector.get("revoke"):
+        from datetime import datetime, timezone
+        store.revoke(RevocationRecord(
+            manifest_id=vector["manifest"]["manifest_id"],
+            revoked_at=datetime.now(timezone.utc),
+            reason="conformance vector",
+            revoked_by="test",
+        ))
+
+    ctx = VerificationContext(**vector["context"])
+    result = verify_manifest(vector["manifest"], ctx, store)
+
+    expected = vector["expected"]
+    assert result.result.value == expected["result"], (
+        f"{vector['id']}: expected {expected['result']}, got {result.result.value}"
+    )
+
+    if "signature_verified" in expected:
+        assert result.signature_verified is expected["signature_verified"], vector["id"]
+    if "attestation_verified" in expected:
+        assert result.attestation_verified is expected["attestation_verified"], vector["id"]
+
+    for field, want in expected.get("fields_verified", {}).items():
+        got = getattr(result.fields_verified, field).value
+        assert got == want, f"{vector['id']}: fields_verified.{field} expected {want}, got {got}"

python/tests/vectors/AM-VEC-001.json

@@ -0,0 +1,76 @@
+{
+  "id": "AM-VEC-001",
+  "description": "All bound artifacts match a signed, in-date manifest.",
+  "spec_refs": [
+    "5.3"
+  ],
+  "manifest": {
+    "manifest_id": "018f4a3b-2c1d-7e5f-a8b9-0d1e2f3a4b5c",
+    "agent_id": "spiffe://trust.example/agent/kyc/prod",
+    "version": "0.1",
+    "issued_at": "2025-01-01T00:00:00Z",
+    "expires_at": "2099-12-31T23:59:59Z",
+    "issuer": "spiffe://trust.example/signing-authority",
+    "crypto_profile": "standard",
+    "artifacts": {
+      "system_prompt": {
+        "hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      },
+      "policy_bundle": {
+        "hash": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+      },
+      "model_identity": {
+        "model_hash": null,
+        "version": "claude-3",
+        "deployment_type": "api"
+      }
+    },
+    "delegation_chain": [],
+    "hitl_record": null,
+    "signature": {
+      "algorithm": "Ed25519",
+      "key_id": "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c",
+      "key_type": "software",
+      "signed_at": "2025-01-01T00:00:00Z",
+      "signature_value": "YUnGWwyv4mUTUrjmXi5FakUaEzPo_JkqEXxT4d6txlN_gFKpXPS2lG52SfuT7trQzwyvc4hwZ28uCg5Ve_0lBg",
+      "signed_fields": [
+        "@context",
+        "@type",
+        "manifest_id",
+        "previous_manifest_id",
+        "agent_id",
+        "version",
+        "min_verifier_version",
+        "issued_at",
+        "expires_at",
+        "issuer",
+        "crypto_profile",
+        "artifacts",
+        "delegation_chain",
+        "hitl_record",
+        "prior_transparency_log_entry",
+        "log_retention",
+        "data_scope",
+        "operational_lifecycle"
+      ]
+    }
+  },
+  "context": {
+    "system_prompt_hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+    "policy_bundle_hash": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+    "model_version": "claude-3",
+    "trusted_keys": {
+      "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c": "A6EHv_POEL4dcN0Y50vAmWfk1jCbpQ1fHdyGZBJVMbg"
+    }
+  },
+  "expected": {
+    "result": "VALID",
+    "signature_verified": true,
+    "fields_verified": {
+      "system_prompt": "MATCH",
+      "policy_bundle": "MATCH",
+      "model_identity": "MATCH",
+      "rag_corpus": "NOT_BOUND"
+    }
+  }
+}

python/tests/vectors/AM-VEC-002.json

@@ -0,0 +1,73 @@
+{
+  "id": "AM-VEC-002",
+  "description": "Runtime system_prompt hash differs from the bound hash.",
+  "spec_refs": [
+    "5.3"
+  ],
+  "manifest": {
+    "manifest_id": "018f4a3b-2c1d-7e5f-a8b9-0d1e2f3a4b5c",
+    "agent_id": "spiffe://trust.example/agent/kyc/prod",
+    "version": "0.1",
+    "issued_at": "2025-01-01T00:00:00Z",
+    "expires_at": "2099-12-31T23:59:59Z",
+    "issuer": "spiffe://trust.example/signing-authority",
+    "crypto_profile": "standard",
+    "artifacts": {
+      "system_prompt": {
+        "hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      },
+      "policy_bundle": {
+        "hash": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+      },
+      "model_identity": {
+        "model_hash": null,
+        "version": "claude-3",
+        "deployment_type": "api"
+      }
+    },
+    "delegation_chain": [],
+    "hitl_record": null,
+    "signature": {
+      "algorithm": "Ed25519",
+      "key_id": "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c",
+      "key_type": "software",
+      "signed_at": "2025-01-01T00:00:00Z",
+      "signature_value": "YUnGWwyv4mUTUrjmXi5FakUaEzPo_JkqEXxT4d6txlN_gFKpXPS2lG52SfuT7trQzwyvc4hwZ28uCg5Ve_0lBg",
+      "signed_fields": [
+        "@context",
+        "@type",
+        "manifest_id",
+        "previous_manifest_id",
+        "agent_id",
+        "version",
+        "min_verifier_version",
+        "issued_at",
+        "expires_at",
+        "issuer",
+        "crypto_profile",
+        "artifacts",
+        "delegation_chain",
+        "hitl_record",
+        "prior_transparency_log_entry",
+        "log_retention",
+        "data_scope",
+        "operational_lifecycle"
+      ]
+    }
+  },
+  "context": {
+    "system_prompt_hash": "sha256:9999999999999999999999999999999999999999999999999999999999999999",
+    "policy_bundle_hash": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+    "model_version": "claude-3",
+    "trusted_keys": {
+      "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c": "A6EHv_POEL4dcN0Y50vAmWfk1jCbpQ1fHdyGZBJVMbg"
+    }
+  },
+  "expected": {
+    "result": "MISMATCH",
+    "signature_verified": true,
+    "fields_verified": {
+      "system_prompt": "MISMATCH"
+    }
+  }
+}

python/tests/vectors/AM-VEC-003.json

@@ -0,0 +1,69 @@
+{
+  "id": "AM-VEC-003",
+  "description": "Manifest expires_at is in the past.",
+  "spec_refs": [
+    "5.3"
+  ],
+  "manifest": {
+    "manifest_id": "018f4a3b-2c1d-7e5f-a8b9-0d1e2f3a4b5c",
+    "agent_id": "spiffe://trust.example/agent/kyc/prod",
+    "version": "0.1",
+    "issued_at": "2025-01-01T00:00:00Z",
+    "expires_at": "2000-01-01T00:00:00Z",
+    "issuer": "spiffe://trust.example/signing-authority",
+    "crypto_profile": "standard",
+    "artifacts": {
+      "system_prompt": {
+        "hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      },
+      "policy_bundle": {
+        "hash": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+      },
+      "model_identity": {
+        "model_hash": null,
+        "version": "claude-3",
+        "deployment_type": "api"
+      }
+    },
+    "delegation_chain": [],
+    "hitl_record": null,
+    "signature": {
+      "algorithm": "Ed25519",
+      "key_id": "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c",
+      "key_type": "software",
+      "signed_at": "2025-01-01T00:00:00Z",
+      "signature_value": "aTYg-k3G99zvZMv5xL7AyHc0ikd7CG9gAMM3jMgvrjWkqhmo8CM6omFi-mOc9hgSJVTywrptIncmuii2Zru-Dg",
+      "signed_fields": [
+        "@context",
+        "@type",
+        "manifest_id",
+        "previous_manifest_id",
+        "agent_id",
+        "version",
+        "min_verifier_version",
+        "issued_at",
+        "expires_at",
+        "issuer",
+        "crypto_profile",
+        "artifacts",
+        "delegation_chain",
+        "hitl_record",
+        "prior_transparency_log_entry",
+        "log_retention",
+        "data_scope",
+        "operational_lifecycle"
+      ]
+    }
+  },
+  "context": {
+    "system_prompt_hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+    "policy_bundle_hash": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+    "model_version": "claude-3",
+    "trusted_keys": {
+      "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c": "A6EHv_POEL4dcN0Y50vAmWfk1jCbpQ1fHdyGZBJVMbg"
+    }
+  },
+  "expected": {
+    "result": "EXPIRED"
+  }
+}

python/tests/vectors/AM-VEC-004.json

@@ -0,0 +1,71 @@
+{
+  "id": "AM-VEC-004",
+  "description": "Manifest id is present in the revocation store.",
+  "spec_refs": [
+    "4.4",
+    "5.3"
+  ],
+  "manifest": {
+    "manifest_id": "018f4a3b-2c1d-7e5f-a8b9-0d1e2f3a4b5c",
+    "agent_id": "spiffe://trust.example/agent/kyc/prod",
+    "version": "0.1",
+    "issued_at": "2025-01-01T00:00:00Z",
+    "expires_at": "2099-12-31T23:59:59Z",
+    "issuer": "spiffe://trust.example/signing-authority",
+    "crypto_profile": "standard",
+    "artifacts": {
+      "system_prompt": {
+        "hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      },
+      "policy_bundle": {
+        "hash": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+      },
+      "model_identity": {
+        "model_hash": null,
+        "version": "claude-3",
+        "deployment_type": "api"
+      }
+    },
+    "delegation_chain": [],
+    "hitl_record": null,
+    "signature": {
+      "algorithm": "Ed25519",
+      "key_id": "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c",
+      "key_type": "software",
+      "signed_at": "2025-01-01T00:00:00Z",
+      "signature_value": "YUnGWwyv4mUTUrjmXi5FakUaEzPo_JkqEXxT4d6txlN_gFKpXPS2lG52SfuT7trQzwyvc4hwZ28uCg5Ve_0lBg",
+      "signed_fields": [
+        "@context",
+        "@type",
+        "manifest_id",
+        "previous_manifest_id",
+        "agent_id",
+        "version",
+        "min_verifier_version",
+        "issued_at",
+        "expires_at",
+        "issuer",
+        "crypto_profile",
+        "artifacts",
+        "delegation_chain",
+        "hitl_record",
+        "prior_transparency_log_entry",
+        "log_retention",
+        "data_scope",
+        "operational_lifecycle"
+      ]
+    }
+  },
+  "context": {
+    "system_prompt_hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+    "policy_bundle_hash": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+    "model_version": "claude-3",
+    "trusted_keys": {
+      "56475aa75463474c0285df5dbf2bcab73da651358839e9b77481b2eab107708c": "A6EHv_POEL4dcN0Y50vAmWfk1jCbpQ1fHdyGZBJVMbg"
+    }
+  },
+  "expected": {
+    "result": "REVOKED"
+  },
+  "revoke": true
+}