diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 986be3b..04e75f1 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -11,6 +11,7 @@ flowchart LR API --> Model["Mock Inference Backend"] API --> Audit["JSONL Audit Sink"] API --> TraceExport["Sanitized Trace Export"] + Policy --> Capacity["Synthetic Capacity Plan"] ``` ## Components @@ -23,6 +24,7 @@ flowchart LR - `gateway/token_budget.py`: deterministic estimated input-token accounting. - `gateway/audit.py`: structured JSONL audit events with identity and trace evidence. - `gateway/trace_exporter.py`: opt-in sanitized trace spans for local OpenTelemetry-shaped evidence. +- `gateway/capacity_plan.py`: synthetic capacity and cost-to-serve planning from model policy and benchmark assumptions. - `gateway/mock_inference.py`: synthetic model response with latency metadata. - `gateway/registry.py`: demo principals and model policies. - `deploy/prometheus` and `deploy/grafana`: local metrics scrape and dashboard provisioning. @@ -35,4 +37,5 @@ flowchart LR - Replace in-memory request and token-budget limiting with Redis or Envoy global rate limits. - Upgrade sanitized trace JSONL to OpenTelemetry SDK export through an OTLP collector while keeping Prometheus metrics for low-cardinality service health. - Capture GPU telemetry from DCGM and attach it to inference metrics. +- Replace synthetic capacity assumptions with measured backend profiles after real model-serving integration exists. - Add per-model data handling rules and prompt/output redaction. diff --git a/README.md b/README.md index 3efcd95..d901650 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Secure GPU Inference Gateway -Security-focused AI infrastructure demo for OIDC/JWT-authenticated model access, role-based authorization, per-model request and token-budget limits, trace-aware audit logs, and policy-driven inference routing. +Security-focused AI infrastructure demo for OIDC/JWT-authenticated model access, role-based authorization, per-model request and token-budget limits, trace-aware audit logs, synthetic capacity/cost planning, and policy-driven inference routing. This repository uses a mock inference backend so the security and infrastructure logic can be reviewed without GPU hardware, model weights, proprietary data, or cloud credentials. @@ -21,6 +21,7 @@ This is the flagship portfolio project for the platform-security-to-AI-infrastru - W3C `traceparent` propagation for OpenTelemetry-compatible request correlation. - Prometheus-compatible `/metrics` endpoint for authentication, policy, limiter, token-throughput, and latency telemetry. - Opt-in sanitized trace JSONL export for local OpenTelemetry-shaped span evidence. +- Synthetic capacity and cost-to-serve planning artifact tied to configured model policies. - Prometheus and Grafana provisioning files for local observability review. - Mock GPU inference backend with latency metadata. - Focused unit tests for policy and limiter behavior. @@ -49,6 +50,7 @@ Relevant areas: - Review `gateway/audit.py` for structured evidence. - Review `gateway/trace_exporter.py` for sanitized trace span export without prompt, output, access-reason, or principal identifiers. - Review `gateway/trace_context.py` for W3C trace context parsing and response propagation. +- Review `gateway/capacity_plan.py` and `artifacts/capacity-plan-evidence.json` for aggregate synthetic capacity and cost-to-serve modeling. - Review `deploy/grafana/dashboards/security-gateway.json` for dashboard queries over the gateway metrics. - Review `docs/OPERATIONS.md` and `deploy/kubernetes/gateway.yaml` for SLO/runbook and deployment thinking. - Check `tests/` for behavior-focused coverage. @@ -85,6 +87,14 @@ uvicorn gateway.app:app --reload Requests still write the normal audit event, but trace export is intentionally narrower. It records service, route, model, outcome, auth method, estimated input-token count, configured token budget, latency, and trace identifiers; it does not record prompt text, model output, access reason, subject, or principal ID. A checked example is in `artifacts/sanitized-trace-evidence.jsonl`. +Capacity plan evidence: + +```bash +python -m gateway.capacity_plan --output artifacts/capacity-plan-evidence.json +``` + +The checked capacity artifact is synthetic aggregate data. It compares configured request and input-token policy limits against modeled per-model request capacity, input-token capacity, decode-token capacity, p95 latency, utilization assumptions, and cost-to-serve estimates. It is meant for review of the planning logic, not as a claim about a production fleet. + Local dashboard stack: ```bash @@ -129,6 +139,7 @@ This project covers: - W3C trace context propagation for request correlation across a model-serving control plane. - Prometheus-compatible metrics for authentication outcomes, policy denials, request/token limiting, input-token throughput, and inference latency. - Sanitized trace export that proves request correlation without leaking prompts, outputs, reasons, or principal identifiers. +- Synthetic capacity and cost-to-serve projection that connects policy budgets to modeled request, token, latency, utilization, and cost assumptions. - Local Prometheus/Grafana review files for model-access, auth, denial, and latency telemetry. - Kubernetes-ready health probes, scrape annotations, and non-root runtime posture. - Backend service design with clear separation between API, policy, rate limiting, and inference. @@ -139,6 +150,7 @@ This project covers: - Replace local HS256 review tokens with JWKS-backed OIDC key rotation. - Replace in-memory request and token-budget limiters with Redis-backed or gateway-level distributed controls. - Upgrade the local trace JSONL proof into full OpenTelemetry SDK export through an OTLP collector, then capture Grafana screenshots from synthetic traffic. +- Replace synthetic capacity inputs with measured backend profiles once a real model-serving adapter exists. - Add policy-as-code examples, redaction controls, and negative authorization tests. - Add CI supply-chain evidence such as SBOM generation, dependency scanning, and container scanning. - Add SOC2/FedRAMP-inspired control mapping notes without claiming certification or production authorization. diff --git a/ROADMAP.md b/ROADMAP.md index 03ab103..8f62da2 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -58,6 +58,7 @@ Status: partially implemented. - Keep the Prometheus `/metrics` endpoint and provisioned Grafana dashboard reviewable under local `docker compose`. - Keep sanitized trace JSONL export opt-in and free of prompt text, model outputs, access reason, subjects, and principal IDs. - Keep estimated input-token budget metrics and audit fields free of prompt text and principal-identifying trace attributes. +- Keep the synthetic capacity plan artifact aggregate-only and tied to configured policy budgets. - Upgrade the trace JSONL proof to full OpenTelemetry SDK export through an OTLP collector. - Add Grafana dashboard screenshots using synthetic traffic. - Add SLO burn-rate alert examples for auth failures, policy denials, rate limiting, and inference latency. @@ -69,6 +70,7 @@ Status: planned. - Add adapters for Triton-compatible, vLLM-compatible, or SGLang-compatible backends. - Keep mock backend tests so the repo remains reviewable without GPU hardware. +- Replace synthetic capacity profiles with measured backend profiles once a real adapter exists. - Add GPU/DCGM telemetry correlation when real backend integration exists. - Extend the current estimated input-token throughput proof with queue depth, backend error rate, and real per-model latency when backend integration exists. diff --git a/artifacts/capacity-plan-evidence.json b/artifacts/capacity-plan-evidence.json new file mode 100644 index 0000000..5e4db35 --- /dev/null +++ b/artifacts/capacity-plan-evidence.json @@ -0,0 +1,62 @@ +{ + "assumptions": { + "cost_basis": "single synthetic accelerator-hour", + "safety_margin": 0.2, + "target_gpu_utilization": 0.72 + }, + "data_scope": "synthetic aggregate benchmark profile; excludes request bodies, decoded text, identities, secrets, and production logs", + "models": [ + { + "accelerator_class": "synthetic-review-gpu", + "cost_per_1000_requests_usd": 0.073457, + "cost_per_1m_input_tokens_usd": 0.278245, + "model_id": "mission-summarizer", + "modeled_decode_tokens_per_minute": 41675.294, + "modeled_input_tokens_per_minute": 167717.647, + "modeled_requests_per_minute": 635.294, + "observed_gpu_utilization": 0.68, + "observed_p95_latency_ms": 410, + "policy_input_token_utilization": 0.047699, + "policy_input_tokens_per_minute": 8000, + "policy_request_utilization": 0.047222, + "policy_requests_per_minute": 30, + "profile_name": "synthetic-standard-summary", + "status": "within_synthetic_capacity" + }, + { + "accelerator_class": "synthetic-review-gpu", + "cost_per_1000_requests_usd": 0.106301, + "cost_per_1m_input_tokens_usd": 0.356867, + "model_id": "threat-triage", + "modeled_decode_tokens_per_minute": 32224.865, + "modeled_input_tokens_per_minute": 130767.568, + "modeled_requests_per_minute": 439.005, + "observed_gpu_utilization": 0.74, + "observed_p95_latency_ms": 530, + "policy_input_token_utilization": 0.045883, + "policy_input_tokens_per_minute": 6000, + "policy_request_utilization": 0.045558, + "policy_requests_per_minute": 20, + "profile_name": "synthetic-restricted-triage", + "status": "within_synthetic_capacity" + }, + { + "accelerator_class": "synthetic-review-gpu", + "cost_per_1000_requests_usd": 0.003788, + "cost_per_1m_input_tokens_usd": 0.022894, + "model_id": "benchmark-echo", + "modeled_decode_tokens_per_minute": 124800.0, + "modeled_input_tokens_per_minute": 873600.0, + "modeled_requests_per_minute": 5280.0, + "observed_gpu_utilization": 0.36, + "observed_p95_latency_ms": 95, + "policy_input_token_utilization": 0.022894, + "policy_input_tokens_per_minute": 20000, + "policy_request_utilization": 0.022727, + "policy_requests_per_minute": 120, + "profile_name": "synthetic-health-check", + "status": "within_synthetic_capacity" + } + ], + "schema_version": 1 +} diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 935606b..e538528 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -11,6 +11,7 @@ This service is intentionally small, but it is shaped like an AI infrastructure - Traceability: every inference response and audit event carries a trace ID for request correlation. - Trace privacy: optional trace exports omit prompt text, generated output, access reason, subject, and principal ID. - Budget visibility: request-count and estimated input-token budget decisions are visible in audit events and metrics. +- Capacity review: model policy changes compare request and input-token limits against a synthetic capacity/cost plan before rollout. ## Metrics @@ -29,6 +30,14 @@ Set `TRACE_EXPORT_PATH` or `OTEL_TRACE_EXPORT_PATH` to write one sanitized JSONL The checked example in `artifacts/sanitized-trace-evidence.jsonl` shows the expected shape. `tests/test_trace_exporter.py` covers the privacy boundary. +## Capacity Plan Artifact + +Run `python -m gateway.capacity_plan --output artifacts/capacity-plan-evidence.json` to regenerate the checked synthetic capacity plan. + +The report compares each configured model policy with a synthetic benchmark profile and records modeled request capacity, input-token capacity, decode-token capacity, p95 latency, target utilization, safety margin, and cost-to-serve estimates. The report is aggregate-only and intentionally excludes request bodies, decoded text, identities, secrets, and production logs. + +Use the artifact before raising `requests_per_minute` or `input_tokens_per_minute`; if a policy would exceed modeled capacity, treat that as a rollout blocker until the benchmark profile or backend fleet shape is updated. + ## Local Dashboard `docker compose up --build` starts the gateway, Prometheus, and Grafana with the dashboard provisioned from `deploy/grafana/dashboards/security-gateway.json`. @@ -92,7 +101,15 @@ Dashboard panels cover: 2. Compare `security_gateway_input_tokens_total` against the model's configured `input_tokens_per_minute`. 3. Review audit events for repeated principal/model pairs, estimated token counts, and shared trace IDs. 4. If traffic is abusive, block or throttle at the edge before raising the model budget. -5. If traffic is legitimate, raise the budget only with a capacity note and rollback time. +5. If traffic is legitimate, regenerate `artifacts/capacity-plan-evidence.json` and raise the budget only with a capacity note and rollback time. + +### Capacity Policy Change + +1. Regenerate `artifacts/capacity-plan-evidence.json`. +2. Confirm the target model status is `within_synthetic_capacity`. +3. Compare `policy_request_utilization` and `policy_input_token_utilization` against the intended rollout margin. +4. Keep the prior policy available for rollback if the change increases either utilization materially. +5. Record the policy diff, modeled capacity result, and rollback owner in the release note. ### Latency Regression @@ -112,4 +129,4 @@ Dashboard panels cover: - Kubernetes readiness and liveness probes should target `/health`. - Prometheus should scrape `/metrics` on port 8000. - Optional trace export path must point at a writable location and must not be treated as a prompt or output sink. -- Model policy changes should review both `requests_per_minute` and `input_tokens_per_minute`. +- Model policy changes should review both `requests_per_minute` and `input_tokens_per_minute`, then regenerate the capacity plan artifact. diff --git a/docs/PORTFOLIO_REVIEW.md b/docs/PORTFOLIO_REVIEW.md index 62ada0e..dffeda4 100644 --- a/docs/PORTFOLIO_REVIEW.md +++ b/docs/PORTFOLIO_REVIEW.md @@ -13,7 +13,9 @@ This project is intentionally designed as a public-safe AI security infrastructu - `gateway/metrics.py`: Prometheus-compatible authentication, policy, limiter, token-throughput, and latency samples. - `gateway/audit.py`: structured JSONL evidence for allowed and denied requests with auth and trace fields. - `gateway/trace_exporter.py`: sanitized trace span export for local observability evidence. +- `gateway/capacity_plan.py`: synthetic capacity and cost-to-serve projection tied to model policies. - `gateway/mock_inference.py`: reviewable model-serving boundary without private models or GPU hardware. +- `artifacts/capacity-plan-evidence.json`: checked aggregate capacity artifact for local review. - `deploy/grafana/dashboards/security-gateway.json`: dashboard queries for request outcomes, latency, auth, denial, and model policy review. - `docs/OPERATIONS.md`: SLOs, alert candidates, and incident runbooks. - `ROADMAP.md`: secure AI / cloud governance roadmap for policy-as-code, redaction, supply-chain evidence, telemetry, and control mapping. @@ -30,6 +32,7 @@ This project is intentionally designed as a public-safe AI security infrastructu - Proving token-budget abuse control without writing prompt text into metrics or trace spans. - Handling request correlation through OpenTelemetry-compatible W3C trace context. - Exporting sanitized trace evidence that keeps prompt, output, access-reason, subject, and principal identifiers out of observability artifacts. +- Connecting model policy limits to synthetic capacity, utilization, latency, and cost-to-serve estimates without using private workload data. - Providing Prometheus/Grafana review files that turn gateway metrics into operational panels. - Designing a mockable inference boundary that can later route to real model-serving backends. - Showing Kubernetes deployment thinking without requiring cloud credentials. @@ -49,6 +52,7 @@ This project is intentionally designed as a public-safe AI security infrastructu - Add mTLS notes for gateway-to-backend communication. - Replace in-memory request/token budget controls with distributed limiters. - Upgrade the local trace JSONL proof into full OpenTelemetry SDK export through an OTLP collector and capture dashboard screenshots from synthetic traffic. +- Replace synthetic capacity profiles with measured backend profiles after model-serving integration exists. - Add policy-as-code and redaction examples with positive and negative test cases. - Add CI supply-chain evidence such as SBOM generation, dependency scanning, and container scanning. - Add SOC2/FedRAMP-inspired control mapping notes without implying certification or production authorization. diff --git a/gateway/capacity_plan.py b/gateway/capacity_plan.py new file mode 100644 index 0000000..ed45cb0 --- /dev/null +++ b/gateway/capacity_plan.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +import argparse +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from gateway.models import ModelPolicy +from gateway.registry import MODEL_POLICIES + + +DEFAULT_TARGET_GPU_UTILIZATION = 0.72 +DEFAULT_SAFETY_MARGIN = 0.20 + + +@dataclass(frozen=True) +class CapacityProfile: + model_id: str + profile_name: str + measured_requests_per_second: float + measured_input_tokens_per_second: float + measured_decode_tokens_per_second: float + observed_p95_latency_ms: float + observed_gpu_utilization: float + gpu_hourly_cost_usd: float + accelerator_class: str = "synthetic-review-gpu" + + def validate(self) -> None: + positive_fields = { + "measured_requests_per_second": self.measured_requests_per_second, + "measured_input_tokens_per_second": self.measured_input_tokens_per_second, + "measured_decode_tokens_per_second": self.measured_decode_tokens_per_second, + "observed_p95_latency_ms": self.observed_p95_latency_ms, + "gpu_hourly_cost_usd": self.gpu_hourly_cost_usd, + } + for field_name, value in positive_fields.items(): + if value <= 0: + raise ValueError(f"{field_name} must be positive") + if not 0 < self.observed_gpu_utilization <= 1: + raise ValueError("observed_gpu_utilization must be in (0, 1]") + + +@dataclass(frozen=True) +class CapacityAssumption: + target_gpu_utilization: float = DEFAULT_TARGET_GPU_UTILIZATION + safety_margin: float = DEFAULT_SAFETY_MARGIN + + def validate(self) -> None: + if not 0 < self.target_gpu_utilization <= 1: + raise ValueError("target_gpu_utilization must be in (0, 1]") + if not 0 <= self.safety_margin < 1: + raise ValueError("safety_margin must be in [0, 1)") + + +DEFAULT_CAPACITY_PROFILES = ( + CapacityProfile( + model_id="mission-summarizer", + profile_name="synthetic-standard-summary", + measured_requests_per_second=12.5, + measured_input_tokens_per_second=3300, + measured_decode_tokens_per_second=820, + observed_p95_latency_ms=410, + observed_gpu_utilization=0.68, + gpu_hourly_cost_usd=2.80, + ), + CapacityProfile( + model_id="threat-triage", + profile_name="synthetic-restricted-triage", + measured_requests_per_second=9.4, + measured_input_tokens_per_second=2800, + measured_decode_tokens_per_second=690, + observed_p95_latency_ms=530, + observed_gpu_utilization=0.74, + gpu_hourly_cost_usd=2.80, + ), + CapacityProfile( + model_id="benchmark-echo", + profile_name="synthetic-health-check", + measured_requests_per_second=55.0, + measured_input_tokens_per_second=9100, + measured_decode_tokens_per_second=1300, + observed_p95_latency_ms=95, + observed_gpu_utilization=0.36, + gpu_hourly_cost_usd=1.20, + ), +) + + +def build_capacity_report( + profiles: tuple[CapacityProfile, ...] = DEFAULT_CAPACITY_PROFILES, + policies: dict[str, ModelPolicy] = MODEL_POLICIES, + assumption: CapacityAssumption = CapacityAssumption(), +) -> dict[str, Any]: + assumption.validate() + estimates = [ + estimate_profile_capacity(profile, policies[profile.model_id], assumption) + for profile in profiles + ] + return { + "schema_version": 1, + "data_scope": "synthetic aggregate benchmark profile; excludes request bodies, decoded text, identities, secrets, and production logs", + "assumptions": { + "target_gpu_utilization": assumption.target_gpu_utilization, + "safety_margin": assumption.safety_margin, + "cost_basis": "single synthetic accelerator-hour", + }, + "models": estimates, + } + + +def estimate_profile_capacity( + profile: CapacityProfile, + policy: ModelPolicy, + assumption: CapacityAssumption, +) -> dict[str, Any]: + profile.validate() + assumption.validate() + utilization_scale = assumption.target_gpu_utilization / profile.observed_gpu_utilization + usable_scale = utilization_scale * (1 - assumption.safety_margin) + modeled_requests_per_second = profile.measured_requests_per_second * usable_scale + modeled_input_tokens_per_second = ( + profile.measured_input_tokens_per_second * usable_scale + ) + modeled_decode_tokens_per_second = ( + profile.measured_decode_tokens_per_second * usable_scale + ) + modeled_requests_per_minute = modeled_requests_per_second * 60 + modeled_input_tokens_per_minute = modeled_input_tokens_per_second * 60 + modeled_decode_tokens_per_minute = modeled_decode_tokens_per_second * 60 + policy_request_utilization = ( + policy.requests_per_minute / modeled_requests_per_minute + ) + policy_input_token_utilization = ( + policy.input_tokens_per_minute / modeled_input_tokens_per_minute + ) + limiting_ratio = max(policy_request_utilization, policy_input_token_utilization) + status = ( + "within_synthetic_capacity" + if limiting_ratio <= 1 + else "exceeds_synthetic_capacity" + ) + + return { + "model_id": profile.model_id, + "profile_name": profile.profile_name, + "accelerator_class": profile.accelerator_class, + "observed_p95_latency_ms": round(profile.observed_p95_latency_ms, 3), + "observed_gpu_utilization": round(profile.observed_gpu_utilization, 6), + "modeled_requests_per_minute": round(modeled_requests_per_minute, 3), + "modeled_input_tokens_per_minute": round(modeled_input_tokens_per_minute, 3), + "modeled_decode_tokens_per_minute": round(modeled_decode_tokens_per_minute, 3), + "policy_requests_per_minute": policy.requests_per_minute, + "policy_input_tokens_per_minute": policy.input_tokens_per_minute, + "policy_request_utilization": round(policy_request_utilization, 6), + "policy_input_token_utilization": round(policy_input_token_utilization, 6), + "cost_per_1000_requests_usd": round( + cost_per_units(profile.gpu_hourly_cost_usd, modeled_requests_per_second, 1000), + 6, + ), + "cost_per_1m_input_tokens_usd": round( + cost_per_units( + profile.gpu_hourly_cost_usd, + modeled_input_tokens_per_second, + 1_000_000, + ), + 6, + ), + "status": status, + } + + +def cost_per_units(hourly_cost: float, units_per_second: float, unit_count: int) -> float: + if hourly_cost <= 0: + raise ValueError("hourly_cost must be positive") + if units_per_second <= 0: + raise ValueError("units_per_second must be positive") + return hourly_cost / (units_per_second * 3600) * unit_count + + +def write_capacity_report(output_path: Path) -> None: + output_path.parent.mkdir(parents=True, exist_ok=True) + report = build_capacity_report() + output_path.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Write a public-safe synthetic gateway capacity plan artifact." + ) + parser.add_argument( + "--output", + default="artifacts/capacity-plan-evidence.json", + help="Path for the generated JSON report.", + ) + args = parser.parse_args() + write_capacity_report(Path(args.output)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_capacity_plan.py b/tests/test_capacity_plan.py new file mode 100644 index 0000000..0a8d410 --- /dev/null +++ b/tests/test_capacity_plan.py @@ -0,0 +1,104 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from gateway.capacity_plan import ( + CapacityAssumption, + CapacityProfile, + build_capacity_report, + cost_per_units, + write_capacity_report, +) +from gateway.registry import MODEL_POLICIES + + +class CapacityPlanTest(unittest.TestCase): + def test_builds_capacity_and_cost_estimates_for_all_policy_models(self) -> None: + report = build_capacity_report() + + self.assertEqual(report["schema_version"], 1) + self.assertEqual( + {model["model_id"] for model in report["models"]}, + set(MODEL_POLICIES), + ) + for model in report["models"]: + self.assertGreater(model["modeled_requests_per_minute"], 0) + self.assertGreater(model["modeled_input_tokens_per_minute"], 0) + self.assertGreater(model["cost_per_1000_requests_usd"], 0) + self.assertGreater(model["cost_per_1m_input_tokens_usd"], 0) + self.assertIn( + model["status"], + {"within_synthetic_capacity", "exceeds_synthetic_capacity"}, + ) + + def test_capacity_status_flags_policy_that_exceeds_synthetic_profile(self) -> None: + profile = CapacityProfile( + model_id="mission-summarizer", + profile_name="tiny-profile", + measured_requests_per_second=0.1, + measured_input_tokens_per_second=10, + measured_decode_tokens_per_second=5, + observed_p95_latency_ms=500, + observed_gpu_utilization=0.9, + gpu_hourly_cost_usd=2.0, + ) + + report = build_capacity_report( + profiles=(profile,), + assumption=CapacityAssumption(target_gpu_utilization=0.5, safety_margin=0.5), + ) + + self.assertEqual( + report["models"][0]["status"], + "exceeds_synthetic_capacity", + ) + + def test_invalid_profiles_are_rejected(self) -> None: + profile = CapacityProfile( + model_id="mission-summarizer", + profile_name="invalid", + measured_requests_per_second=1, + measured_input_tokens_per_second=1, + measured_decode_tokens_per_second=1, + observed_p95_latency_ms=1, + observed_gpu_utilization=0, + gpu_hourly_cost_usd=1, + ) + + with self.assertRaisesRegex(ValueError, "observed_gpu_utilization"): + build_capacity_report(profiles=(profile,)) + + def test_cost_per_units_validates_positive_inputs(self) -> None: + with self.assertRaisesRegex(ValueError, "hourly_cost"): + cost_per_units(0, 10, 1000) + with self.assertRaisesRegex(ValueError, "units_per_second"): + cost_per_units(1, 0, 1000) + + def test_written_artifact_is_public_safe_aggregate_data(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + output_path = Path(tmpdir) / "capacity-plan-evidence.json" + + write_capacity_report(output_path) + + artifact = output_path.read_text(encoding="utf-8") + parsed = json.loads(artifact) + self.assertEqual(parsed["schema_version"], 1) + forbidden_fragments = ( + "access_reason", + "auth_subject", + "credential", + "generated_text", + "model_output", + "principal_id", + "prompt", + "raw_input", + "subject", + ) + lowered = artifact.lower() + for fragment in forbidden_fragments: + self.assertNotIn(fragment, lowered) + + +if __name__ == "__main__": + unittest.main()