From c1944a53f092e1ce489149f6ee1520f64484d027 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 6 Nov 2026 23:17:11 -0800 Subject: [PATCH 01/41] fix(aws-startup-advisor): refresh GPT-5.4/GPT-5.5 Bedrock endpoint metadata from the public model cards The plugin recorded GPT-5.4 and GPT-5.5 as bedrock-mantle-only (no CRIS, no inference profile, no bedrock-runtime surface) with Chat Completions "unconfirmed", a 272K context window and no long-context price tier, and the canonical catalog cited "internal reference" probes as provenance. The public model cards no longer say any of that. This PR corrects the metadata only; it does not change endpoint selection, dispatch, or add new code paths. Sources (read 2026-11-06 via the AWS Documentation MCP tools): - https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html - https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html What the cards state, now recorded verbatim in the catalog and prose: - Endpoints: bedrock-runtime supported (Responses, Chat Completions, Converse; not Messages/Invoke) and bedrock-mantle supported (Responses, Chat Completions). Runtime has no in-region form; requests name us.openai.gpt-5.4 / global.openai.gpt-5.4 and us.openai.gpt-5.5 / global.openai.gpt-5.5. Mantle in-region: 5.4 = us-east-1, us-east-2, us-west-2, us-gov-west-1; 5.5 = us-east-1, us-east-2. US Geo CRIS from 6 regions, Global CRIS from 33. - Context 1,050,000 tokens shared, 128,000 max output. - Pricing (Standard, In-Region only; Priority/Flex unsupported): 5.4 short 2.75 / 0.275 cache read / 16.50, long 5.50 / 0.55 / 24.75, GovCloud short 3.375 / 0.3375 / 20.25; 5.5 short 5.50 / 0.55 / 33.00, long 11.00 / 1.10 / 49.50. No Geo/Global price rows, so prose keeps the ~10% figure for these two on every inference option and never claims parity. - Runtime features: invocation logs, streaming, Guardrails and application inference profiles (Converse only), implicit prompt caching and structured outputs (Responses only); no server-side tool use. Changes: - skills/shared/ai/models/openai-bedrock.json: GPT-5.4/5.5 rows re-sourced to the public cards (no "internal reference"), mantle_openai_chat available, runtime_converse path with CRIS, inference_profiles (us/global), region_availability, pricing tiers, context/output limits; path-level constraints corrected. - Canonical shared docs: ai-openai-to-bedrock.md (Tier 0 table, region gate, caching, Bedrock-native section, volume guidance), ai-migration-guardrails.md, schema-design-aws-ai.md (ban is on BARE ids paired with converse/runtime, not on the models), bedrock-pricing-cache.md (rows, long-context table, caching note, GovCloud rate). - gcp-to-aws and azure-to-aws: references/shared/openai-on-bedrock.md, shared/schema-design-aws-ai.md (azure), design-ai.md, design.md/generate.md _assert lines (azure), generate-artifacts-ai.md (azure), generate-ai.md, discover-preview.md and clarify-ai-only.md (gcp), clarify-ai-only.md (azure). - bedrock-migration-assessment clarify-ai-only.md; agent-advisor model-recommend.md (documents the engine's current GPT-5.x reshape as an engine limitation, not a model fact). - llm-to-bedrock: resolve-bedrock-model-id.md (Case A/B), behavior-delta openai-to-bedrock.md, SKILL.md (two id-form sentences); docstrings/comments in preflight_bedrock.py, iam_policy.py, bedrock_pricing.py and the preflight 404 remedy string. The is_mantle_model / is_mantle_gpt ID-form logic is unchanged. - agents: llm2bedrock-code-rewriter.md (Converse rule is id-form-dependent), llm2bedrock-report-generator.md (cost text), llm2bedrock-prompt-evaluator.md. - openai_model_recommendation.py: only the chat_completions_to_responses block message text (dropped the "GPT-5.x on Mantle rejects Chat Completions" claim). - Tests: tests/test_gpt6_review_round2.py, test_gpt6_review_round5.py, test_gpt6_round4_findings.py and agent-advisor scripts/test_openai_model_recommendation.py now assert the corrected public facts (CRIS ids present, "mantle-only"/"no CRIS"/"unconfirmed" absent, catalog provenance is the public card). New regression tests in test_preflight_bedrock.py, test_iam_policy.py, test_bedrock_pricing.py prove us./global.openai.gpt-5.4|5.5 classify as runtime and bare ids as mantle in all three helpers. - Vendored copies under agent-advisor, gcp-to-aws, azure-to-aws re-synced with tools/sync-vendored.py. Validation (all run from the worktree with mise Python 3.12): - mise -C plugins/aws-startup-advisor run ci: PASS - lint:md 0 errors; lint:model-id OK (19 retired IDs enforced); lint:model-staleness fresh (4d old, 45d window); lint:model-catalog-drift OK (3 pre-existing warnings: sonnet-5-5, haiku-5-5, gpt-6.1-terra not in catalog; unrelated); lint:startup-model-pricing OK; lint:artifacts PASS (99 checked, 0 errors); lint:vendored-parity PASS (62 files byte-identical); lint:skill-refs PASS (1482 checked, 0 missing); lint:consent-disclosure OK; test: 1395 passed, 222 subtests passed; test:telemetry: 311 passed; test:heroku-source-review: 44 pass, 0 fail - agent-advisor pytest (skills/agent-advisor/scripts): 564 passed, 0 failed (562 at base + 2 new) - llm-to-bedrock pytest (skills/llm-to-bedrock/scripts): 418 passed, 0 failed (415 at base + 3 new; needs --with httpx --with aws-bedrock-token-generator) - root: python3 tools/validate.py "All validations passed"; mise run lint:md 0 issues / 2763 files; lint:manifests, lint:skill-parity, lint:spec all pass; test:tools 6 OK. mise run build's security step fails on a pre-existing gitleaks false positive in upstream commit f8d09fd1 (tests/test_bedrock_migration_assessment_discover.py:118, "claude-3.5-sonnet"); gitleaks protect over this change set: no leaks found. - grep over plugins/ for mantle-only / "no CRIS" / "no inference profile" / "no runtime path" / "unconfirmed" / "internal reference" / "no 1M tier" near GPT-5.4/5.5: no remaining claims (only negative test assertions and the bare-id rule "bare proprietary openai.gpt-* IDs use Mantle only"). Base: 0a218f8cc13220013f72147e2e0fdbe350b34ae2 (upstream/main) --- .../agents/llm2bedrock-code-rewriter.md | 2 +- .../agents/llm2bedrock-prompt-evaluator.md | 7 +- .../agents/llm2bedrock-report-generator.md | 2 +- .../phases/model-recommend/model-recommend.md | 16 +++- .../vendored/ai/bedrock-pricing-cache.md | 36 +++++--- .../vendored/ai/models/openai-bedrock.json | 89 ++++++++++++++---- .../scripts/openai_model_recommendation.py | 5 +- .../test_openai_model_recommendation.py | 42 ++++++++- .../phases/clarify/clarify-ai-only.md | 7 +- .../references/phases/design/design-ai.md | 28 ++++-- .../references/phases/design/design.md | 2 +- .../phases/generate/generate-artifacts-ai.md | 7 +- .../references/phases/generate/generate.md | 2 +- .../references/shared/openai-on-bedrock.md | 91 +++++++++++++------ .../references/shared/schema-design-aws-ai.md | 30 +++--- .../vendored/ai/ai-migration-guardrails.md | 8 +- .../vendored/ai/ai-openai-to-bedrock.md | 57 +++++++----- .../vendored/ai/bedrock-pricing-cache.md | 36 +++++--- .../vendored/ai/schema-design-aws-ai.md | 23 +++-- .../phases/clarify/clarify-ai-only.md | 4 +- .../phases/clarify/clarify-ai-only.md | 4 +- .../references/phases/design/design-ai.md | 35 ++++--- .../phases/discover/discover-preview.md | 4 +- .../references/phases/generate/generate-ai.md | 4 +- .../references/shared/openai-on-bedrock.md | 91 +++++++++++++------ .../vendored/ai/ai-migration-guardrails.md | 8 +- .../vendored/ai/ai-openai-to-bedrock.md | 57 +++++++----- .../vendored/ai/bedrock-pricing-cache.md | 36 +++++--- .../vendored/ai/schema-design-aws-ai.md | 23 +++-- .../skills/llm-to-bedrock/SKILL.md | 7 +- .../references/openai-to-bedrock.md | 4 +- .../resolve-bedrock-model-id.md | 39 ++++---- .../llm-to-bedrock/scripts/bedrock_pricing.py | 15 ++- .../llm-to-bedrock/scripts/iam_policy.py | 6 +- .../scripts/preflight_bedrock.py | 17 ++-- .../scripts/test_bedrock_pricing.py | 12 +++ .../llm-to-bedrock/scripts/test_iam_policy.py | 12 +++ .../scripts/test_preflight_bedrock.py | 25 ++++- .../shared/ai/ai-migration-guardrails.md | 8 +- .../skills/shared/ai/ai-openai-to-bedrock.md | 57 +++++++----- .../skills/shared/ai/bedrock-pricing-cache.md | 36 +++++--- .../shared/ai/models/openai-bedrock.json | 89 ++++++++++++++---- .../skills/shared/ai/schema-design-aws-ai.md | 23 +++-- .../tests/test_gpt6_review_round2.py | 29 ++++-- .../tests/test_gpt6_review_round5.py | 17 +++- .../tests/test_gpt6_round4_findings.py | 30 ++++-- 46 files changed, 794 insertions(+), 388 deletions(-) diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md b/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md index 6cfdc6979..d4065c4a5 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md @@ -257,7 +257,7 @@ client = anthropic.Anthropic( Preserve compatible SDK request/response shapes. A model version change still requires §9 target checks, including typed response parsing for Opus 5.5. Apply the Chat Completions → Responses reshape only when that API change was selected. Preserve the exact validated runtime profile ID or ARN. After applying the changes above, skip the Converse-specific guidance in the rest of §8; the §9 behavior-delta application still applies normally. -**Converse for a proprietary GPT target is family-dependent.** GPT-5.5 / GPT-5.4 (`openai.gpt-5.5`, `openai.gpt-5.4`) have no `bedrock-runtime` surface — a boto3 `converse()` call against them fails at runtime; if your context pairs one with the Converse path, stop and report the contradiction. GPT-5.6 and GPT-6 (Astra, Sol, Luna, 6.1 Sol) DO have a `bedrock-runtime` path via CRIS ids (`us.`/`in.`/`global.` prefixed): when the plan says `migration_path: runtime_openai_cris`, a Converse or Responses rewrite against the CRIS id is valid (base URL `bedrock-runtime.{region}.amazonaws.com/openai/v1` for the OpenAI-compatible APIs; note Guardrails are Converse-only and prompt caching Responses-only on runtime). For a Responses-based source app, the mantle express lane above remains the default and smallest change. +**Converse for a proprietary GPT target is id-form-dependent, not family-dependent.** A **bare** proprietary GPT id (`openai.gpt-5.5`, `openai.gpt-5.4`, `openai.gpt-5.6-*`, `openai.gpt-6-*`) is a `bedrock-mantle` id with no `bedrock-runtime` in-region form — a boto3 `converse()` call against a bare id fails at runtime; if your context pairs a bare id with the Converse path, stop and report the contradiction. GPT-5.6, GPT-6 (Astra, Sol, Luna, 6.1 Sol) **and GPT-5.5 / GPT-5.4** (model cards read 2026-11-06: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`; Responses, Chat Completions and Converse on runtime, no Invoke) all have a `bedrock-runtime` path via CRIS ids (`us.`/`in.`/`global.` prefixed): when the plan says `migration_path: runtime_openai_cris`, a Converse, Responses or Chat Completions rewrite against the CRIS id is valid (base URL `bedrock-runtime.{region}.amazonaws.com/openai/v1` for the OpenAI-compatible APIs; note Guardrails are Converse-only and prompt caching Responses-only on runtime). For a Responses-based source app, the mantle express lane above remains the default and smallest change. ### Converse rewrite (default) diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-prompt-evaluator.md b/plugins/aws-startup-advisor/agents/llm2bedrock-prompt-evaluator.md index d18e1a13d..d933c4b3e 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-prompt-evaluator.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-prompt-evaluator.md @@ -229,12 +229,13 @@ Interpret the result: region. These models are **in-region only**, so there is no cross-region inference profile to fall back to and `resolve-bedrock-model-id`'s inference-profile ranking does not apply. Return - `{ blocked: { reason: 'model_unresolvable', detail: ' — is not available on the bedrock-mantle endpoint in . Mantle is in-region only; for Astra use Virginia/Oregon Standard Mantle or a supported us./global. runtime profile with an explicit plan change; GPT-5.6 may use its us./in./global. CRIS forms, while GPT-5.5/5.4 need a supported region or a different model.' } }`. + `{ blocked: { reason: 'model_unresolvable', detail: ' — is not available on the bedrock-mantle endpoint in . Mantle is in-region only; for Astra use Virginia/Oregon Standard Mantle or a supported us./global. runtime profile with an explicit plan change; GPT-5.6 may use its us./in./global. CRIS forms and GPT-5.5/5.4 their us./global. CRIS forms with an explicit plan change, or pick a supported mantle region or a different model.' } }`. Use `model_unresolvable` rather than a new reason — its recovery path (user picks or pastes an ID, recorded in `resolved_model_overrides`) is exactly right here. Astra may use only supported `us.` / `global.` CRIS candidates; check its own region matrix. For GPT-5.6 a `us.`/`in.`/`global.`-prefixed CRIS candidate is legitimate - (it switches the run to the Converse path); for GPT-5.5 / GPT-5.4 no prefixed - form exists — do not offer one. + (it switches the run to the Converse path); for GPT-5.5 / GPT-5.4 the `us.`/`global.` + prefixed forms exist on `bedrock-runtime` (model cards read 2026-11-06) and are offered + the same way — never silently, always as an explicit plan change. - **HTTP 401 / 403** — distinguish the two causes from the message. If it names model access, return `{ blocked: { reason: 'model_access', ... } }` as in §6. Otherwise it is IAM: the principal needs the `bedrock-mantle:*` actions diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md index 54d09601e..9b9de321e 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md @@ -143,7 +143,7 @@ The pricing script only covers Bedrock models. For the source provider (OpenAI, **Astra:** use its separately dated Bedrock rates and context/inference tier. Its source price is absent from the table below; verify that price or mark the source comparison unavailable. Do not estimate Astra from a lower GPT tier or assume GPT-5.x parity/premium. -**GPT-5.x same-model pair cost depends on the inference option — state it conditionally, never as a flat figure.** In-Region and Geo CRIS are priced at OpenAI's _data residency_ tier (1.10x standard), so those moves cost ~10% more — quote the increase plainly and carry the non-cost justification (commitments, governance, residency, prompt caching, no behaviour delta). **GPT-5.6 on Global CRIS is priced at the standard rate — cost parity** — applicable only when the workload has no data-residency constraint; when the plan's `migration_path` is `runtime_openai_cris` with a `global.` id, report parity, not a premium. GPT-5.5/5.4 have no CRIS, so the ~10% figure always applies to them. **If a pair's workload exceeds 272K context**, price the Bedrock side at the GPT-5.6 long-context tier — 2.0x input, 1.5x output per option — and show that line separately; GPT-5.5/5.4 have no long-context tier so their usable window is 272K. +**GPT-5.x same-model pair cost depends on the inference option — state it conditionally, never as a flat figure.** In-Region and Geo CRIS are priced at OpenAI's _data residency_ tier (1.10x standard), so those moves cost ~10% more — quote the increase plainly and carry the non-cost justification (commitments, governance, residency, prompt caching, no behaviour delta). **GPT-5.6 on Global CRIS is priced at the standard rate — cost parity** — applicable only when the workload has no data-residency constraint; when the plan's `migration_path` is `runtime_openai_cris` with a `global.` id, report parity, not a premium. GPT-5.5/5.4 do have `us.`/`global.` CRIS ids on `bedrock-runtime`, but their model cards (read 2026-11-06) publish In-Region pricing only — so the ~10% figure applies to them on every inference option until a Global rate is published; never report parity for them. **If a pair's workload exceeds 272K context**, price the Bedrock side at the long-context tier and show that line separately: GPT-5.6 is 2.0x input / 1.5x output per option; GPT-5.5 is 11.00 / 1.10 cache read / 49.50 and GPT-5.4 is 5.50 / 0.55 cache read / 24.75 per 1M tokens (In-Region, from their cards), applied to all tokens of the request, within a 1,050,000-token context window. If the source model is not in this table (likely for any model released after the table's last update — check the provider's public pricing page if you know current rates), note it as a gap in the Risk Assessment section, label the figure "(estimated from ``)", and estimate using the closest listed model's pricing. diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md index f3e4563cb..c418842ed 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md @@ -94,17 +94,23 @@ reuse the Anthropic `preserve_messages_api` switch): OpenAI is handled by a dedicated provider module (`openai-bedrock.json` catalog). Recognized GPT-5.6 and GPT-6 source identities are evaluated against their exact same-model -catalog entry before capability/numeric fallbacks. GPT-5.x on Mantle is Responses-only, so a Chat -Completions source is reshaped rather than routed to `mantle_openai_chat`. GPT-6 Chat Completions -sources with required or preferred API continuity instead keep that surface through -`mantle_openai_chat`; GPT-6 Responses sources use `mantle_openai_responses`. Both Mantle paths are +catalog entry before capability/numeric fallbacks. The engine routes only recognized GPT-6 Chat +Completions sources to `mantle_openai_chat`; a GPT-5.x Chat Completions source is currently +reshaped to Responses. That reshape is a current engine limitation, not a model fact: the catalog +records Chat Completions as supported on Mantle for GPT-5.5 and GPT-5.4 (model cards read +2026-11-06) and as unchecked for GPT-5.6 Sol. GPT-6 Chat Completions sources with required or +preferred API continuity keep that surface through `mantle_openai_chat`; GPT-6 Responses sources +use `mantle_openai_responses`. Both Mantle paths are in-region only and enforce the selected model's cataloged `region_availability`: Astra reaches us-east-1 and us-west-2, while Sol, Luna, and 6.1 Sol reach us-east-1 only. Outside that footprint, the engine skips the same-model Mantle candidate and evaluates its existing capability/numeric fallbacks. Governance requirements select the exact same GPT-5.6 or GPT-6 model through `runtime_converse` only when the requested Global or geography-scoped CRIS profile is cataloged for that model and source region; an unsupported profile falls through to an eligible documented -runtime candidate or remains an explicit decision. GPT-5.5/5.4 remain mantle-only. GPT-6 Astra's +runtime candidate or remains an explicit decision. GPT-5.5/5.4 also carry a cataloged +`runtime_converse` path via `us.`/`global.` CRIS ids (model cards read 2026-11-06); the engine does +not yet recognize them as same-model source identities, so a GPT-5.5/5.4 source under governance +requirements still lands on the Claude Converse tier until that recognition is added. GPT-6 Astra's runtime surface lacks `Invoke` support (Responses/Chat Completions/Converse only), while Sol/Luna/6.1 Sol support all four APIs. Every selected path remains provisional until it is probed in the target account and region. Azure OpenAI remains an explicit `provider_module_pending` diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md index 18266733c..a4206583c 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md @@ -133,8 +133,8 @@ See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommen | GPT-5.6 Sol | openai.gpt-5.6-sol | OpenAI | 4.40 | 22.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 8.80/33.00) | | GPT-5.6 Terra | openai.gpt-5.6-terra | OpenAI | 2.20 | 13.20 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 4.40/19.80) | | GPT-5.6 Luna | openai.gpt-5.6-luna | OpenAI | 0.22 | 1.32 | 272K | fast | active (mantle in-region + runtime CRIS; 1M tier 0.44/1.98) | -| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle only; no 1M tier) | -| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle only; no 1M tier) | +| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 11.00/49.50; card read 2026-11-06) | +| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 5.50/24.75; card read 2026-11-06) | | Gemma 4 31B | google.gemma-4-31b | Google | 0.14 | 0.40 | 256K | mid | active (Mantle only) | | Gemma 4 26B-A4B | google.gemma-4-26b-a4b | Google | 0.13 | 0.40 | 256K | efficient | active (Mantle only) | | Gemma 4 E2B | google.gemma-4-e2b | Google | 0.04 | 0.08 | 128K | budget | active (Mantle only) | @@ -353,17 +353,24 @@ Regions (mantle): us-east-1 / us-east-2 for all five; us-west-2 additionally car | GPT-5.5 | 5.50 | 33.00 | — | 0.55 | | GPT-5.4 | 2.75 | 16.50 | — | 0.275 | -#### Long context (1M) — GPT-5.6 only, 2.0x input / 1.5x output - -| Model | Input $/1M | Output $/1M | -| ------------- | ---------- | ----------- | -| GPT-5.6 Sol | 8.80 | 33.00 | -| GPT-5.6 Terra | 4.40 | 19.80 | -| GPT-5.6 Luna | 0.44 | 1.98 | - -A workload above 272K context must be priced at the long-context tier. GPT-5.5 and GPT-5.4 have no such tier, so -their usable window is 272K. Global CRIS is published at OpenAI's standard list price (cost parity, GPT-5.6 only — -see `openai-on-bedrock.md`). GovCloud GPT-5.4 is 3.30 / 19.80. +#### Long context (>272K input tokens) — GPT-5.6 at 2.0x input / 1.5x output; GPT-5.5 / GPT-5.4 per their model cards + +| Model | Input $/1M | Output $/1M | Cache read $/1M | +| ------------- | ---------- | ----------- | --------------- | +| GPT-5.6 Sol | 8.80 | 33.00 | — | +| GPT-5.6 Terra | 4.40 | 19.80 | — | +| GPT-5.6 Luna | 0.44 | 1.98 | — | +| GPT-5.5 | 11.00 | 49.50 | 1.10 | +| GPT-5.4 | 5.50 | 24.75 | 0.55 | + +A workload above 272K input tokens must be priced at the long-context tier, applied to all input and output tokens +of the request. GPT-5.5 and GPT-5.4 rows read from their model cards on 2026-11-06 +([GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), +[GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)); both cards state a +1,050,000-token context window (128,000 max output) and list `us.`/`global.` CRIS ids on `bedrock-runtime`, but +publish the In-Region inference option only. Global CRIS is published at OpenAI's standard list price (cost parity, +GPT-5.6 only — see `openai-on-bedrock.md`). GovCloud (US-West) GPT-5.4 short-context is 3.375 / 20.25 (cache read +0.3375) per the 2026-11-06 card; an earlier revision recorded 3.30 / 19.80. > **One source conflict.** These models are absent > from the AWS Price List API entirely: querying `AmazonBedrock` returns only `gpt-oss` and GPT OSS Safeguard, and @@ -373,7 +380,8 @@ see `openai-on-bedrock.md`). GovCloud GPT-5.4 is 3.30 / 19.80. **Prompt caching (GPT-5.6 only):** cached input read at a 90% discount, cache write at 1.25x uncached input, minimum 1,024-token prefix, up to 4 breakpoints, ≥30-minute retention. Cached input is exempt from the input-TPM quota. -GPT-5.5 and GPT-5.4 do not list prompt caching support. +GPT-5.5 and GPT-5.4 list only implicit prompt caching on `bedrock-runtime` Responses (automatic, no request +parameters; cache-read rate above, no cache-write rate) and none on mantle (model cards read 2026-11-06). #### Open-weight gpt-oss diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json index a6bb34385..bed2a584b 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json @@ -3,8 +3,8 @@ "provider": "openai", "verified_at": "2026-09-24", "verified_region": "us-east-2", - "source": "Opus 5.5 cross-family target: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html (retrieved 2026-09-24). OpenAI and other unchanged entries retain their evidence from the superseded 2026-08-21 catalog (removed; see git history).", - "disclaimer": "The Opus cross-family target is updated to Opus 5.5; same-model OpenAI paths and their earlier evidence remain unchanged. This is a migration target catalog, not an account-availability claim. Verify the chosen model, API, region and CRIS profile before implementation. Opus 5.5 uses CRIS on runtime outside eu-west-2, has 1M context/128K output, always-on adaptive thinking, and no Batch tier. Fable and Mythos remain excluded from default selection. Opus 5.5 is the only Opus model in the automatic selection pool; older Opus snapshots and pricing records are historical references. The runtime in-region exception in eu-west-2 was verified 2026-10-03; the original catalog and other model evidence dates remain unchanged.", + "source": "Opus 5.5 cross-family target: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html (retrieved 2026-09-24). GPT-5.5 and GPT-5.4 rows re-read from their public model cards on 2026-11-06 (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html): both now list bedrock-runtime (Responses, Chat Completions, Converse via us./global. CRIS ids) and Chat Completions on bedrock-mantle. Other OpenAI entries retain their evidence from the superseded 2026-08-21 catalog (removed; see git history).", + "disclaimer": "The Opus cross-family target is updated to Opus 5.5; the GPT-5.5 and GPT-5.4 entries were re-read from their public model cards on 2026-11-06 (runtime CRIS paths, Chat Completions on mantle, 1.05M context, long-context pricing); the other same-model OpenAI paths and their earlier evidence remain unchanged. This is a migration target catalog, not an account-availability claim. Verify the chosen model, API, region and CRIS profile before implementation. Opus 5.5 uses CRIS on runtime outside eu-west-2, has 1M context/128K output, always-on adaptive thinking, and no Batch tier. Fable and Mythos remain excluded from default selection. Opus 5.5 is the only Opus model in the automatic selection pool; older Opus snapshots and pricing records are historical references. The runtime in-region exception in eu-west-2 was verified 2026-10-03; the original catalog and other model evidence dates remain unchanged.", "paths": { "mantle_openai_responses": { "sdk": "openai", @@ -19,8 +19,8 @@ "GPT-5.x on Bedrock Mantle is served through the Responses API" ], "constraints": [ - "Mantle itself has no cross-region form for any GPT model (in-region only); for GPT-5.6 the bedrock-runtime CRIS path covers additional regions instead (see runtime_converse)", - "GPT-5.x on Mantle rejects Chat Completions; a Chat Completions source must reshape to Responses (reference-verified on gpt-5.4; the 5.6 cards list Chat Completions as a supported API but its mantle-side availability is unprobed — verify in the target account)", + "Mantle itself has no cross-region form for any GPT model (in-region only); for GPT-5.6, GPT-5.5 and GPT-5.4 the bedrock-runtime CRIS path covers additional regions instead (see runtime_converse)", + "Chat Completions support on Mantle is model-specific: the GPT-5.4 and GPT-5.5 model cards list Chat Completions as supported on bedrock-mantle (read 2026-11-06); the GPT-5.6 Sol card does not. A Chat Completions source reshapes to Responses only when the selected model's card does not list Chat Completions — verify in the target account", "Availability must be probed in the target account and region" ] }, @@ -36,7 +36,7 @@ "OpenAI-compatible Chat Completions surface for models that accept it" ], "constraints": [ - "Not available for the GPT-5.x models listed here: rejected on gpt-5.4 (internal reference probe) and unsupported on the gpt-5.6-sol model card", + "Model-specific: listed as supported on bedrock-mantle by the GPT-5.4 and GPT-5.5 model cards (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, read 2026-11-06) and by the GPT-6 cards; unsupported on the gpt-5.6-sol model card. Uses /openai/v1/chat/completions", "Availability and model support must be probed in the target account and region" ] }, @@ -51,10 +51,10 @@ "strengths": [ "Bedrock-native multi-model request surface", "Supports Bedrock Guardrails (Converse API only), invocation logging, and CloudWatch", - "For GPT-5.6 this is a SAME-MODEL governance path via CRIS (us./in./global. ids; base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs); the model cards recommend runtime for new applications, and Global CRIS is priced at OpenAI's standard rate (cost parity)" + "For GPT-5.6, GPT-5.5 and GPT-5.4 this is a SAME-MODEL governance path via CRIS (us./in./global. ids; base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs); the model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" ], "constraints": [ - "GPT-5.5 / GPT-5.4 have NO runtime path — a Converse target for them means a Bedrock-native model and a boto3 rewrite of the OpenAI integration", + "GPT-5.5 / GPT-5.4 on runtime are CRIS-only (us. and global. ids; no in-region form) per their model cards (read 2026-11-06); the cards publish In-Region (1.10x) pricing only, so do not assume Global CRIS price parity for them", "GPT-5.6 on runtime is CRIS-only (no in-region form); Geo CRIS stays within the geography, Global CRIS routes anywhere — pick by data-residency posture. Runtime does not support server-side tool use, structured outputs, or application inference profiles for these models; prompt caching is Responses-only", "Bedrock-native models generally require a Global or geography-scoped CRIS inference profile", "Availability must be probed in the target account and region" @@ -111,19 +111,47 @@ "generation": "reasoning", "version": "5.5", "class": "reasoning", - "context_window": 272000, - "output_token_ceiling": "unknown", + "context_window": 1050000, + "output_token_ceiling": 128000, + "inference_profiles": { + "us": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1"], + "global": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1", "mx-central-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ap-southeast-5", "ap-southeast-6", "ap-southeast-7", "il-central-1", "me-central-1", "me-south-1", "af-south-1", "sa-east-1"] + }, + "region_availability": { + "commercial": ["us-east-1", "us-east-2"], + "govcloud": [], + "evidence": "bedrock-mantle in-region rows of the GPT-5.5 model card, read 2026-11-06: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html" + }, + "pricing": { + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-06); USD per 1M tokens, Standard tier; the card lists the In-Region inference option only", + "commercial_short_context_le_272k": {"input": 5.50, "cache_read": 0.55, "output": 33.00}, + "commercial_long_context_gt_272k": {"input": 11.00, "cache_read": 1.10, "output": 49.50}, + "service_tiers": {"standard": true, "priority": false, "flex": false, "reserved": false} + }, "capabilities": [ "reasoning", - "tool_or_function_calling" + "tool_or_function_calling", + "image_input_vision" ], - "capability_source": "AWS model card 2026-07-21 (launch 2026-06-01, Active): advanced coding/agentic tasks; tool calling via the Responses API's built-in tool use. Feature-level Mantle behavior not probed — verify before relying. Re-verified 2026-08-21: still mantle-only, in-region only; no CRIS, no runtime path, no long-context tier.", + "capability_source": "AWS model card https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-06; launch 2026-06-01, Active, EOL no sooner than 2027-06-01): text+image input, text output; 1,050,000-token context shared between input and output, 128,000 max output. Mantle lists server-side and client-side tool calling; runtime lists invocation logs, streaming, Guardrails (Converse only), application inference profiles (Converse only), implicit prompt caching (Responses only), structured outputs (Responses) and reasoning summaries; server-side tool use is unsupported on runtime. Documentation claims, not account probes — verify before relying.", "paths": { "mantle_openai_responses": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": false, - "evidence": "AWS model card model-card-openai-gpt-55 (2026-07-21): bedrock-mantle Programmatic Access with Responses sample code; Geo/Global inference not supported" + "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1 and us-east-2 only; Geo/Global not supported on mantle" + }, + "mantle_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.5", + "requires_cris": false, + "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code" + }, + "runtime_converse": { + "available": true, + "model_id": "openai.gpt-5.5", + "requires_cris": true, + "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.5 (Geo) or global.openai.gpt-5.5 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs" } } }, @@ -213,27 +241,50 @@ "generation": "reasoning", "version": "5.4", "class": "reasoning", - "context_window": "unknown", - "output_token_ceiling": "unknown", + "context_window": 1050000, + "output_token_ceiling": 128000, + "inference_profiles": { + "us": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1"], + "global": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1", "mx-central-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ap-southeast-5", "ap-southeast-6", "ap-southeast-7", "il-central-1", "me-central-1", "me-south-1", "af-south-1", "sa-east-1"] + }, + "region_availability": { + "commercial": ["us-east-1", "us-east-2", "us-west-2"], + "govcloud": ["us-gov-west-1"], + "evidence": "bedrock-mantle in-region rows of the GPT-5.4 model card, read 2026-11-06: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html" + }, + "pricing": { + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-06); USD per 1M tokens, Standard tier; the card lists the In-Region inference option only", + "commercial_short_context_le_272k": {"input": 2.75, "cache_read": 0.275, "output": 16.50}, + "commercial_long_context_gt_272k": {"input": 5.50, "cache_read": 0.55, "output": 24.75}, + "govcloud_us_west_short_context_le_272k": {"input": 3.375, "cache_read": 0.3375, "output": 20.25}, + "service_tiers": {"standard": true, "priority": false, "flex": false, "reserved": false} + }, "capabilities": [ "tool_or_function_calling", "structured_output_json", "reasoning", - "streaming" + "streaming", + "image_input_vision" ], - "capability_source": "internal reference smoke tests and its reasoning-API migration notebook (Responses path, 2026-06 probes) — the only entry with feature-level Mantle behavior probes (tools, structured output, sampling); limits not stated Re-verified 2026-08-21: still mantle-only, in-region only; no CRIS, no runtime path, no long-context tier.", + "capability_source": "AWS model card https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-06; launch 2026-06-01, Active, EOL no sooner than 2027-06-01): text+image input, text output; 1,050,000-token context shared between input and output, 128,000 max output. Mantle lists server-side and client-side tool calling; runtime lists invocation logs, response streaming, Guardrails (Converse only), application inference profiles (Converse only), implicit prompt caching (Responses only), structured outputs (JSON Schema; Responses API) and reasoning summaries; server-side tool use is unsupported on runtime. Documentation claims, not account probes — verify before relying.", "paths": { "mantle_openai_responses": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": false, - "evidence": "internal reference smoke-test default Mantle model id; client.responses.create verified live" + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1, us-east-2, us-west-2 and us-gov-west-1; Geo/Global not supported on mantle" }, "mantle_openai_chat": { - "available": false, + "available": true, "model_id": "openai.gpt-5.4", "requires_cris": false, - "evidence": "internal reference verified gotcha: GPT-5.x on Mantle is Responses-only; Chat Completions rejected" + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code" + }, + "runtime_converse": { + "available": true, + "model_id": "openai.gpt-5.4", + "requires_cris": true, + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.4 (Geo) or global.openai.gpt-5.4 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs" } } }, diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py index a7f1289cf..e3b1a07e0 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py @@ -1271,8 +1271,9 @@ def _unresolved( ( "The selected Astra Responses path requires reshaping the Chat source." if model_key == "openai_gpt_6_astra" - else "GPT-5.x on Mantle rejects Chat Completions; the source must reshape " - "to the Responses API." + else "The selected Mantle Responses path requires reshaping the Chat source " + "to the Responses API (Chat Completions support is per-model: see the " + "selected model's mantle_openai_chat catalog entry)." ), "Apply the request/response/tool/state reshape deltas before cutover.", ) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py index ef92d7291..7d22b84a7 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py @@ -64,18 +64,58 @@ def test_responses_source_with_continuity_recommends_mantle_responses(): assert rec["primary_model"] == "openai.gpt-5.6-sol" -def test_chat_completions_gpt5_target_selects_responses_not_chat(): +def test_chat_completions_unrecognized_gpt5_source_lands_on_sol_responses(): + # The default workload's source is `gpt-5.4`, which the engine does not yet + # recognize as a same-model identity (see _SOURCE_MODEL_KEYS), so it lands on + # the first catalog candidate — GPT-5.6 Sol — whose card leaves Chat + # Completions unchecked on Mantle. The reshape here follows from THAT model's + # card, not from a blanket "GPT-5.x on Mantle is Responses-only" rule: the + # GPT-5.4 card itself lists Chat Completions on Mantle (see the catalog test + # below). rec = _recommend( _workload( source={"api_surface": "chat_completions"}, requirements={"api_continuity": "required"}, ) ) + assert rec["primary_model"] == "openai.gpt-5.6-sol" + assert OPENAI_CATALOG["models"]["openai_gpt_5_6_sol"]["paths"]["mantle_openai_chat"]["available"] is False assert rec["api_path"] == "mantle_openai_responses" assert rec["api_path"] != "mantle_openai_chat" assert "chat_completions_to_responses_required" in _codes(rec["blocks"]) +@pytest.mark.parametrize( + ("catalog_key", "model_id", "card"), + [ + ("openai_gpt_5_4", "openai.gpt-5.4", "model-card-openai-gpt-54.html"), + ("openai_gpt_5_5", "openai.gpt-5.5", "model-card-openai-gpt-55.html"), + ], +) +def test_catalog_gpt54_gpt55_record_public_runtime_and_chat_facts(catalog_key, model_id, card): + # Public model cards (read 2026-11-06): both models list bedrock-runtime + # (Responses, Chat Completions, Converse via us./global. CRIS ids; no in-region + # form), Chat Completions on bedrock-mantle, a 1,050,000-token context window + # and 128,000 max output. Provenance must be the public card, never an + # internal reference. + model = OPENAI_CATALOG["models"][catalog_key] + paths = model["paths"] + assert paths["mantle_openai_responses"]["available"] is True + assert paths["mantle_openai_chat"]["available"] is True + assert paths["runtime_converse"]["available"] is True + assert paths["runtime_converse"]["requires_cris"] is True + assert paths["runtime_converse"]["model_id"] == model_id + assert f"us.{model_id}" in paths["runtime_converse"]["evidence"] + assert f"global.{model_id}" in paths["runtime_converse"]["evidence"] + assert set(model["inference_profiles"]) == {"us", "global"} + assert model["context_window"] == 1050000 + assert model["output_token_ceiling"] == 128000 + assert card in model["capability_source"] + for path in paths.values(): + assert "internal reference" not in path["evidence"] + assert "internal reference" not in model["capability_source"] + + def test_chat_completions_case_includes_full_reshape_deltas(): rec = _recommend( _workload( diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md index 9dd6a3cb4..760e5e5bb 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md @@ -129,9 +129,10 @@ running). → `ai_constraints.cross_cloud`. Default `"latency-acceptable"`. | Claude (Anthropic SDK) | Same model on Bedrock | client swap only — no model change | | Other / multiple / don't know | ask / infer | — | -**Same-model rows first.** GPT-5.6/5.5/5.4 run on Bedrock via `bedrock-mantle` (Responses, -in-region only) — the case is AWS commitments/governance/residency, not savings, and not parity -(~10% above OpenAI standard). For sources with no Bedrock equivalent, present a same-vendor +**Same-model rows first.** GPT-5.6/5.5/5.4 run on Bedrock via `bedrock-mantle` (Responses or +Chat Completions, in-region only) or via `bedrock-runtime` CRIS ids (`us.`/`global.`) — the case +is AWS commitments/governance/residency, not savings, and not parity (~10% above OpenAI standard +in-region; GPT-5.6 Global CRIS is at the standard rate). For sources with no Bedrock equivalent, present a same-vendor upgrade AND a cross-family option. → `ai_model_baseline`. **After Batch 1:** interpret answers, write `preferences-draft.json` (`metadata.draft: true`, diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md index 1d05cd4a5..7b4564837 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md @@ -155,7 +155,9 @@ data-residency-tier premium, while supported Global CRIS uses standard-rate pari Report the actual option; do not downgrade solely for the modest verified premium. **GPT-5.6/GPT-6 only:** above 272K context, apply 2.0x input and 1.5x output rates only when the selected model and inference option support that tier; the resulting gap may change -the assessment. GPT-5.5/5.4 have no 1M tier. **Opus 5.5:** use the selected region/profile's +the assessment. GPT-5.5/5.4 have their own long-context (>272K) rates on their model cards +(read 2026-11-06; see `openai-on-bedrock.md` § Pricing) — use those, not the GPT-5.6 +multiplier. **Opus 5.5:** use the selected region/profile's flat rates from `references/vendored/ai/bedrock-pricing-cache.md` throughout the full 1M context; do not apply the GPT multiplier. Verify other models' tiers individually. @@ -216,15 +218,21 @@ key/token provider), model ID, and IAM (`bedrock-mantle:*`) change. Read `references/shared/openai-on-bedrock.md` for exact values. **A Chat Completions source is a supported near-drop-in for GPT-6 and GPT-5.6** (both list Chat Completions on mantle and runtime CRIS per their model cards) — keep `chat.completions.create` unless a Responses-only feature or -demonstrated constraint drives a reshape. **GPT-5.5 and GPT-5.4 are the exception**: Chat -Completions support is unconfirmed on their model cards, so a Chat Completions source on either -still needs a planned reshape to `responses.create`. **No Converse fallback for GPT-5.5/GPT-5.4 — GPT-5.6 and -GPT-6 differ.** GPT-5.5 and GPT-5.4 are mantle-only, in-region only: needing Guardrails/Knowledge -Bases/logging/an unsupported region forces a `converse` path with `model_change: true` to a -Bedrock-native model (or `gpt-oss`). GPT-5.6 Sol/Terra/Luna and all five GPT-6 models (Astra, Sol, -Luna, 6.1 Sol, 6.1 Terra) DO have a `bedrock-runtime` CRIS path (`us.`/`global.` prefixed IDs) — for these -models the same need is satisfied by switching endpoint, not model: record -`migration_path: "runtime_openai_cris"` with `model_change: false`. Note: `runtime_openai_cris` +demonstrated constraint drives a reshape. **GPT-5.5 and GPT-5.4 follow the same rule**: their +model cards (read 2026-11-06) list Chat Completions as supported on both `bedrock-mantle` and +`bedrock-runtime`, so a Chat Completions source on either may keep `chat.completions.create`; the +reshape to `responses.create` is optional for them too. Plan a reshape only when the selected +model's card leaves Chat Completions unchecked on the selected endpoint. **Every proprietary GPT +model on Bedrock has a `bedrock-runtime` CRIS path — including GPT-5.5 and GPT-5.4.** GPT-5.6 +Sol/Terra/Luna, all five GPT-6 models (Astra, Sol, Luna, 6.1 Sol, 6.1 Terra), and GPT-5.5 / +GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / +`global.openai.gpt-5.4`; Responses, Chat Completions and Converse on runtime, no runtime +in-region form) have a `bedrock-runtime` CRIS path (`us.`/`global.` prefixed IDs) — for all of +them a Guardrails/logging/application-inference-profile/unsupported-region need is satisfied by +switching endpoint, not model: record `migration_path: "runtime_openai_cris"` with +`model_change: false`. Reserve `converse` with `model_change: true` to a Bedrock-native model (or +`gpt-oss`) for a need the same-model CRIS path cannot serve (Knowledge Bases, intelligent prompt +routing). Note: `runtime_openai_cris` for GPT-6 Astra is valid only via Responses/Chat Completions/Converse — Astra does not support `Invoke`, so verify per-feature against Astra's model card before assuming parity with Sol/Luna/6.1 Sol. **When Guardrails is the driving need and the design keeps diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design.md index 382da53e4..60dda47b0 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design.md @@ -82,7 +82,7 @@ _postconditions: _on_failure: _halt_and_inform - _assert: "WHEN aws-design-ai.json exists: every design_blocks[] row has exactly one of target_bedrock_model / target_aws_service non-null (XOR), and every document_extraction/image_analysis/speech_transcription row has target_bedrock_model null, a non-null target_aws_service, and honest_assessment 'not_applicable' — per references/shared/schema-design-aws-ai.md" _on_failure: _halt_and_inform - - _assert: "WHEN aws-design-ai.json exists AND summary.ai_source is azure_openai or openai: every source model that is available on Bedrock and carried by the target region maps to itself with model_change false — not to a Claude/Nova substitute; and no proprietary GPT-5.5 or GPT-5.4 model ID is paired with a converse/bedrock-runtime migration path (bare proprietary openai.gpt-* IDs use Mantle only; runtime_openai_cris accepts documented Astra us./global. and GPT-5.6/GPT-6 profiles with caller-region checks and the selected API; GPT-5.6 and GPT-6 models MAY use migration_path: runtime_openai_cris on bedrock-runtime — this is not a violation)" + - _assert: "WHEN aws-design-ai.json exists AND summary.ai_source is azure_openai or openai: every source model that is available on Bedrock and carried by the target region maps to itself with model_change false — not to a Claude/Nova substitute; and no bare proprietary openai.gpt-* model ID (including bare openai.gpt-5.5 or openai.gpt-5.4) is paired with a converse/bedrock-runtime migration path (bare proprietary openai.gpt-* IDs use Mantle only; runtime_openai_cris accepts documented Astra us./global., GPT-5.6/GPT-6 and GPT-5.5/GPT-5.4 us./global. profiles with caller-region checks and the selected API; GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 models MAY use migration_path: runtime_openai_cris on bedrock-runtime — this is not a violation)" _on_failure: _halt_and_inform - _assert: "WHEN aws-design-ai.json exists: regional_warnings[] and multi_model_warnings[] are present (empty is fine); agentic_design is present only when agentic_profile.is_agentic is true; and App Runner appears nowhere" _on_failure: _halt_and_inform diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md index 646840dd2..109a9a003 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -155,9 +155,10 @@ is AWS-only and ports from gcp unchanged. - [ ] No secret VALUE in any emitted file; credentials are env-var references. - [ ] Bare proprietary GPT ids use Mantle; Astra runtime calls retain the validated `us.` / `global.` CRIS id. Chat/Responses and runtime artifact selection match the saved design. -- [ ] No **GPT-5.5 or GPT-5.4** model ID paired with a Converse/`bedrock-runtime` path (those two - models only — mantle-only, no CRIS). GPT-5.6 and GPT-6 models MAY use - `migration_path: "runtime_openai_cris"` on `bedrock-runtime` — this is NOT a violation. +- [ ] No **bare** proprietary GPT model ID (including bare `openai.gpt-5.5` / `openai.gpt-5.4`) + paired with a Converse/`bedrock-runtime` path. GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 MAY use + `migration_path: "runtime_openai_cris"` on `bedrock-runtime` with a `us.`/`in.`/`global.` + CRIS id — this is NOT a violation. - [ ] Plugin attribution strings say "Azure-to-AWS". - [ ] `generation-ai.json` validates; `rollback_plan.mechanism == "feature_flag"`. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate.md index be0ac3018..94730ad32 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate.md @@ -73,7 +73,7 @@ _postconditions: _on_failure: _halt_and_inform - _assert: "no secret VALUE from the inventory appears in any generated artifact; secrets are emitted as Secrets Manager references" _on_failure: _halt_and_inform - - _assert: "WHEN aws-design-ai.json exists AND run_mode is decide_and_execute: generation-ai.json exists and validates, its rollback_plan.mechanism is 'feature_flag' with flag_name 'AI_PROVIDER' and default_value 'azure_openai'; an ai-migration/ directory was produced (setup_bedrock.sh, test_comparison.py, bedrock_monitoring.tf on every path; migrate_to_mantle.sh for the mantle path OR provider_adapter.* for the direct/gpt-oss path, not both); and no proprietary GPT-5.5 or GPT-5.4 model ID is paired with a converse/bedrock-runtime path (bare proprietary openai.gpt-* IDs use Mantle only; runtime_openai_cris accepts documented Astra us./global. and GPT-5.6/GPT-6 profiles with caller-region checks and the selected API; GPT-5.6 and GPT-6 models MAY use migration_path: runtime_openai_cris on bedrock-runtime — this is not a violation). When aws-design-ai.json is absent this is vacuously satisfied" + - _assert: "WHEN aws-design-ai.json exists AND run_mode is decide_and_execute: generation-ai.json exists and validates, its rollback_plan.mechanism is 'feature_flag' with flag_name 'AI_PROVIDER' and default_value 'azure_openai'; an ai-migration/ directory was produced (setup_bedrock.sh, test_comparison.py, bedrock_monitoring.tf on every path; migrate_to_mantle.sh for the mantle path OR provider_adapter.* for the direct/gpt-oss path, not both); and no bare proprietary openai.gpt-* model ID (including bare openai.gpt-5.5 or openai.gpt-5.4) is paired with a converse/bedrock-runtime path (bare proprietary openai.gpt-* IDs use Mantle only; runtime_openai_cris accepts documented Astra us./global., GPT-5.6/GPT-6 and GPT-5.5/GPT-5.4 us./global. profiles with caller-region checks and the selected API; GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 models MAY use migration_path: runtime_openai_cris on bedrock-runtime — this is not a violation). When aws-design-ai.json is absent this is vacuously satisfied" _on_failure: _halt_and_inform _forbids_files: - azure-resource-inventory.json diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md index 3b7b1e153..7c9cddaa9 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md @@ -2,6 +2,7 @@ **Last verified:** 2026-10-10 **GPT-6.1 Terra verified:** 2026-10-27 +**GPT-5.5 / GPT-5.4 re-verified:** 2026-11-06 against their model cards ([GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), [GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)) — both now list `bedrock-runtime` (Responses, Chat Completions, Converse via `us.`/`global.` CRIS ids), Chat Completions on mantle, a 1.05M context window and a long-context price tier **Sources:** [OpenAI model cards](https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html) (per-model cards linked below), [GPT-5.6 launch post](https://aws.amazon.com/blogs/machine-learning/get-started-with-openai-gpt-5-6-sol-terra-and-luna-on-amazon-bedrock/), [GPT-5.6 GA announcement](https://aws.amazon.com/about-aws/whats-new/2026/07/openai-gpt-sol-terra/), @@ -32,8 +33,8 @@ risk) all defer to it. Do not restate model IDs, regions, or endpoint paths else | GPT-5.6 Sol | `openai.gpt-5.6-sol` | Jul 13, 2026 | 1M | Active | [card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html) | | GPT-5.6 Terra | `openai.gpt-5.6-terra` | Jul 13, 2026 | 1M | Active | [card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html) | | GPT-5.6 Luna | `openai.gpt-5.6-luna` | Jul 13, 2026 | 1M | Active | [card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html) | -| GPT-5.5 | `openai.gpt-5.5` | Jun 1, 2026 | 272K | Active | [card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html) | -| GPT-5.4 | `openai.gpt-5.4` | Jun 1, 2026 | 272K | Active | [card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html) | +| GPT-5.5 | `openai.gpt-5.5` | Jun 1, 2026 | 1.05M | Active | [card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html) | +| GPT-5.4 | `openai.gpt-5.4` | Jun 1, 2026 | 1.05M | Active | [card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html) | | gpt-oss-120b | `openai.gpt-oss-120b` | Aug 5, 2025 | 128K | Active | open-weight; also on `bedrock-runtime` as `openai.gpt-oss-120b-1:0` | | gpt-oss-20b | `openai.gpt-oss-20b` | Aug 5, 2025 | 128K | Active | open-weight; also on `bedrock-runtime` as `openai.gpt-oss-20b-1:0` | | GPT OSS Safeguard | `openai.gpt-oss-safeguard-120b` / `-20b` | — | — | Active | content-moderation / guardrail enforcement, not general chat | @@ -47,8 +48,10 @@ share a generation with a future `Sol`. **GPT-6** carries forward `Sol` and `Lun discount) — not a new tier name (cite [openai.com/index/introducing-gpt-6-1-sol/](https://openai.com/index/introducing-gpt-6-1-sol/)). > **Context-window conflict (resolved):** the GPT-5.6 launch blog states 272K for all three variants; all three -> model cards state 1M. **The model cards are authoritative** — use 1M for GPT-5.6. GPT-5.5 and GPT-5.4 are 272K on -> both sources. Re-check on refresh; if AWS corrects the blog, the cards still win. +> model cards state 1M. **The model cards are authoritative** — use 1M for GPT-5.6. The GPT-5.5 and GPT-5.4 cards +> (read 2026-11-06) state 1,050,000 tokens shared between input and output with 128,000 max output; the 272K figure +> recorded earlier is now only their short-/long-context pricing breakpoint. Re-check on refresh; if AWS corrects +> the blog, the cards still win. **Not on Bedrock (as of this refresh):** GPT-4o, GPT-4.1, GPT-4 / GPT-4 Turbo, GPT-3.5 Turbo, the o-series (o1/o3/o4-mini), GPT-5 / GPT-5.1 / GPT-5.2, and the `*-Pro` variants (GPT-5.5 Pro, GPT-5.4 Pro). Sources whose model @@ -112,8 +115,12 @@ input and text output. The launch announcement rounds the input context to 1M; u exact limits for capacity checks. Browser/computer-use capability does not establish parity with OpenAI-hosted tools; probe the application's specific tool flow. -**GPT-5.5 and GPT-5.4 are `bedrock-mantle`-only and in-region only.** Their model cards list a single -Programmatic Access row (`bedrock-mantle`, Geo/Global "Not supported") and In-Region pricing only. +**GPT-5.5 and GPT-5.4 have TWO endpoints** (re-verified 2026-11-06 against their model cards; earlier revisions of +this file recorded them as `bedrock-mantle`-only). Both Programmatic Access tables now carry a `bedrock-runtime` +row (In-Region "Not supported"; Geo `us.openai.gpt-5.5` / `us.openai.gpt-5.4`; Global `global.openai.gpt-5.5` / +`global.openai.gpt-5.4`) and a `bedrock-mantle` row (bare id, in-region only). Supported APIs: `Responses`, +`Chat Completions` and `Converse` on `bedrock-runtime`; `Responses` and `Chat Completions` on `bedrock-mantle`; +`Messages` and `Invoke` on neither. The pricing tables still publish the In-Region inference option only. **GPT-5.6 Sol / Terra / Luna have TWO endpoints** (verified 2026-08-21 — this changed after this file's original 2026-08-10 verification; the Refresh Checklist predicted it): @@ -132,13 +139,16 @@ Constraints that still break naive assumptions: mantle path explicitly, and the runtime example gives `bedrock-runtime.{region}.amazonaws.com/openai/v1`. 2. **The endpoints take different model-id forms.** Mantle takes the bare `openai.gpt-5.6-*` id and has no CRIS; `bedrock-runtime` takes ONLY a CRIS inference-profile id (`us.` / `in.` / `global.` prefixed) and has no - in-region form. `bedrock:ListInferenceProfiles` returns the 5.6 CRIS profiles; it never returns the bare mantle - ids, and it returns nothing for GPT-5.5 / GPT-5.4. + in-region form. `bedrock:ListInferenceProfiles` returns the CRIS profiles (5.6, and `us.`/`global.` for + GPT-5.5 / GPT-5.4 per their cards); it never returns the bare mantle ids. 3. **API surfaces differ by endpoint.** The 5.6 cards list `Responses`, `Chat Completions`, `Invoke`, and `Converse` as supported APIs, with runtime-side feature splits: **Guardrails are Converse-only; prompt caching is Responses-only on runtime**; server-side tool use, structured outputs, and application inference profiles are NOT - supported on runtime (server-side tool calling IS supported on mantle). For GPT-5.5 / GPT-5.4 treat Responses on - mantle as the only verified surface. + supported on runtime (server-side tool calling IS supported on mantle). The GPT-5.5 / GPT-5.4 cards (read + 2026-11-06) list `Responses`, `Chat Completions` and `Converse` on runtime (no `Invoke`) and `Responses` and + `Chat Completions` on mantle; on runtime they list Guardrails and application inference profiles as + Converse-only, and implicit prompt caching and structured outputs as Responses-only. Chat Completions is a + documented surface for GPT-5.5 / GPT-5.4 on both endpoints — probe it in the target account like any other. ### GPT-6 family: Ultrafast tier @@ -189,8 +199,10 @@ that expires within 12 hours and is not refreshed. Do not recommend it for produ **IAM:** on the mantle path, the managed policy `AmazonBedrockMantleInferenceAccess` grants what inference needs, including `bedrock-mantle:CreateInference` and `bedrock-mantle:CallWithBearerToken` — `bedrock:InvokeModel` does not -authorize mantle calls. On the GPT-5.6 `bedrock-runtime` path the usual `bedrock:InvokeModel*` against the CRIS -inference-profile ARN applies, as for any other runtime model. +authorize mantle calls. On the GPT-5.6 / GPT-5.5 / GPT-5.4 `bedrock-runtime` path the usual `bedrock:InvokeModel*` +against the CRIS inference-profile ARN applies, as for any other runtime model; the GPT-5.5 / GPT-5.4 cards add +that the identity also needs `bedrock:InvokeModel` on the account's default project +(`arn:aws:bedrock:{region}:{account-id}:project/default`). **Reasoning effort:** the five GPT-5.x models accept `none`, `low`, `medium`, `high`, `xhigh`, `max`. Because these models reason before responding, the model's output items (which may include reasoning items) must be passed back in the next @@ -200,7 +212,8 @@ request for multi-turn and tool-calling flows. ## Regional Availability — Endpoint-Aware -**The mantle in-region matrix** (the only reach for GPT-5.5 / GPT-5.4, and the in-region option for GPT-5.6): +**The mantle in-region matrix** (the in-region option for every proprietary GPT model; GPT-5.5 / GPT-5.4 rows +re-verified 2026-11-06): | Model | us-east-1 | us-east-2 | us-west-2 | us-gov-west-1 | us-gov-east-1 | | ------------- | --------- | --------- | --------- | ------------- | ------------- | @@ -230,8 +243,16 @@ any GPT-6 model, it just means using the runtime endpoint with a CRIS id. **GPT-5.6 additionally reaches most commercial regions via `bedrock-runtime` CRIS** (Geo `us.` / `in.`, Global `global.` inference profiles; the Sol card's runtime footprint spans 30+ regions). So a region outside the mantle matrix does NOT make the same-model path unavailable for GPT-5.6 — it means using the runtime endpoint with a CRIS -id, with the data-residency implications of cross-region routing. For GPT-5.5 / GPT-5.4 the mantle matrix is a hard -gate: no CRIS, no fallback. +id, with the data-residency implications of cross-region routing. + +**GPT-5.5 / GPT-5.4 runtime availability** (model cards read 2026-11-06): US Geo CRIS (`us.openai.gpt-5.5` / +`us.openai.gpt-5.4`) can be called from `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `ca-central-1` and +`ca-west-1`. Global CRIS (`global.openai.gpt-5.5` / `global.openai.gpt-5.4`) is listed in those six plus +`mx-central-1`, every EU region (`eu-central-1/2`, `eu-north-1`, `eu-south-1/2`, `eu-west-1/2/3`), `ap-east-2`, +`ap-northeast-1/2/3`, `ap-south-1/2`, `ap-southeast-1` through `-7`, `il-central-1`, `me-central-1`, `me-south-1`, +`af-south-1` and `sa-east-1` (33 regions). No runtime in-region form, no EU/India Geo profile, and no GovCloud +runtime row. So the mantle matrix is no longer a hard gate for GPT-5.5 / GPT-5.4; only a strictly in-region +requirement outside the mantle matrix blocks the same-model path. Verify current footprints per model card / `aws___get_regional_availability` (AWS MCP Server) — the CRIS lists move faster than this file. @@ -247,7 +268,8 @@ inference-option dimension:** workload has no data-residency constraint. So the honest cost statement is conditional, not flat: a same-model GPT-5.6 move on Global CRIS is -**cost-neutral**; the same move in-region or Geo (and any GPT-5.5 / GPT-5.4 move) is **~10% more expensive**. The +**cost-neutral**; the same move in-region or Geo is **~10% more expensive**. The GPT-5.5 / GPT-5.4 cards publish +In-Region pricing only (no Geo/Global rows as of 2026-11-06), so quote ~10% for them on every inference option. The same Global-CRIS-is-cost-neutral / in-Region-and-Geo-is-~10%-more pattern applies to the GPT-6 family below. Never state either number without stating the inference option it belongs to. @@ -302,12 +324,15 @@ provider rate before claiming savings or parity for the selected context tier. A workload above 272K context must be priced at the long-context tier. -### GPT-5.5 / GPT-5.4 — In-Region only (no CRIS, no long-context tier; usable window 272K) +### GPT-5.5 / GPT-5.4 — In-Region pricing only; short- and long-context tiers (model cards read 2026-11-06) + +The cards publish only the In-Region inference option (the `us.` / `global.` runtime CRIS ids exist but carry no +separate price row). Long-context rates apply to all input and output tokens of a request above 272K input tokens. -| Model | In-Region (in · out) | Cache read | Notes | -| ------- | -------------------- | ---------- | -------------------------------- | -| GPT-5.5 | 5.50 · 33.00 | 0.55 | no cache-write rate published | -| GPT-5.4 | 2.75 · 16.50 | 0.275 | GovCloud (US-West): 3.30 · 19.80 | +| Model | Short context ≤272K (in · cache read · out) | Long context >272K (in · cache read · out) | Notes | +| ------- | ------------------------------------------- | ------------------------------------------ | ---------------------------------------- | +| GPT-5.5 | 5.50 · 0.55 · 33.00 | 11.00 · 1.10 · 49.50 | no cache-write rate published | +| GPT-5.4 | 2.75 · 0.275 · 16.50 | 5.50 · 0.55 · 24.75 | GovCloud (US-West) short: 3.375 · 0.3375 · 20.25 | > **The Luna "blog discrepancy" resolved differently than first recorded.** The AWS News Blog's 0.20 / 1.20 is not > an error — it is the **Global CRIS** rate, now published on the Luna card. An earlier revision of this file said @@ -323,8 +348,10 @@ Astra has separate Responses-only caching evidence above. Do not inherit GPT-5.6 Listed as a supported feature on the GPT-5.6 Sol/Terra/Luna model cards and on all five GPT-6 model cards (Astra, Sol, Luna, 6.1 Sol, 6.1 Terra — confirmed by the cache-write/cache-read columns in the GPT-6 pricing tables above). The -GPT-5.5 and GPT-5.4 cards list client-side tool calling in that slot instead and do **not** list prompt caching. -Do not assume caching on 5.5/5.4. +GPT-5.5 and GPT-5.4 cards (read 2026-11-06) list **implicit prompt caching on `bedrock-runtime` Responses only** +(automatic, no request parameters) with a cache-read rate and no cache-write rate; explicit caching is unsupported +and the mantle feature list carries client-side tool calling in that slot instead. Do not assume caching on +5.5/5.4 mantle, and do not inherit the GPT-5.6 prefix/breakpoint rules below for them. | Property | Value | | ----------------- | ------------------------------------------------------------ | @@ -352,10 +379,10 @@ This corrects two claims that were previously applied to all Mantle traffic in t - There is **no shared 10,000 RPM account limit** governing these models — the quota dimension is TPM, per model, per region. -- "Switch to `bedrock-runtime`" is **not a throughput remedy for GPT-5.5 / GPT-5.4** — they have no - `bedrock-runtime` path at all. For GPT-5.6 the runtime path DOES exist (CRIS only; see Access Paths above), so - moving there is a legitimate option — but treat it as an endpoint/architecture choice with its own quota family - and residency implications, not a free throughput escape hatch. +- "Switch to `bedrock-runtime`" is **not a free throughput escape hatch**. For GPT-5.6, GPT-5.5 and GPT-5.4 the + runtime path exists (CRIS only; see Access Paths above — GPT-5.5 / GPT-5.4 re-verified 2026-11-06), so moving + there is a legitimate option — but treat it as an endpoint/architecture choice with its own quota family and + residency implications. The supported mitigations are: exponential backoff with a bounded retry count (`max_retries` on the OpenAI SDK), spreading load across minutes rather than bursting, ramping request rate gradually, and prompt caching (cached input @@ -396,8 +423,8 @@ This model family is moving fast (two GA waves and a repricing inside 10 weeks). 1. Re-read the [OpenAI model card index](https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html) for models added or removed, and each per-model card for lifecycle state and EOL date. -2. Recheck the region matrix AND the GPT-5.6 CRIS footprints — for 5.5/5.4 a region change is migration-blocking; - for 5.6 the runtime/CRIS path usually covers the region instead. +2. Recheck the region matrix AND the CRIS footprints for every proprietary GPT model — the runtime/CRIS path + usually covers a region outside the mantle matrix; only a strictly in-region requirement there is blocking. 3. Recheck rates on the Bedrock pricing page OpenAI tab, and resolve any row still marked _unverified_. 4. Recheck whether the Price List API has gained GPT-5.x coverage; if it has, drop the caveat above and let `estimate-ai.md` price these models from the MCP. @@ -412,3 +439,9 @@ This model family is moving fast (two GA waves and a repricing inside 10 weeks). 2026), and GPT-6.1 Sol (GA Sep 29, 2026) are all confirmed on Bedrock with their own model cards and pricing tables. New discovery this refresh: a 6x-Standard **Ultrafast** service tier on Astra and 6.1 Sol only (Sol and Luna remain Standard-only) — the first new tier shape since this file started tracking the family. +9. **2026-11-06 refresh — GPT-5.5 / GPT-5.4 re-read.** Item 5 fired again: both cards now list `bedrock-runtime` + (Responses, Chat Completions, Converse via `us.` / `global.` CRIS ids; no in-region runtime form), Chat + Completions on `bedrock-mantle`, a 1,050,000-token context window with 128,000 max output, a long-context + (>272K) price tier, implicit prompt caching on runtime Responses, and a GovCloud (US-West) GPT-5.4 rate of + 3.375 · 20.25. Earlier "mantle-only / no CRIS / 272K / no long-context tier" claims for these two models were + stale and are corrected throughout this file. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md index 811b7c92e..36a049656 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md @@ -123,17 +123,20 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` ## migration_path vocabulary -`mantle_openai_chat` preserves a selected GPT-6 Chat Completions API on Mantle. +`mantle_openai_chat` preserves a selected Chat Completions API on Mantle for a model whose card +lists it there (GPT-6, GPT-5.5, GPT-5.4). `mantle_openai_responses` (Azure OpenAI / OpenAI source, model on Bedrock, region carries it — -keep the SDK, `model_change: false`) · `runtime_openai_cris` (GPT-5.6/GPT-6 same-model move -served via `bedrock-runtime` CRIS, `us.`/`global.` prefixed, `model_change: false`) · `converse` -(Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when **GPT-5.5 or GPT-5.4 -specifically** must move off mantle for Guardrails / Knowledge Bases / logging / an unsupported -region, or when the GPT-6/5.6 CRIS path itself cannot serve the need) · `gpt-oss` (OpenAI-lineage -model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK swap). **GPT-5.5 -and GPT-5.4** are NEVER paired with a `converse`/`bedrock-runtime` path — those two models only -are mantle-only, no CRIS. GPT-5.6 and GPT-6 models MAY use `runtime_openai_cris` on -`bedrock-runtime` — this is not a violation. Note: `runtime_openai_cris` for GPT-6 Astra is valid +keep the SDK, `model_change: false`) · `runtime_openai_cris` (GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 +same-model move served via `bedrock-runtime` CRIS, `us.`/`in.`/`global.` prefixed, +`model_change: false`) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set +`model_change: true` when the workload needs Knowledge Bases, intelligent prompt routing, or +another feature the same-model CRIS path itself cannot serve) · `gpt-oss` (OpenAI-lineage model +on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK swap). A **bare** +proprietary `openai.gpt-*` id (including bare `openai.gpt-5.5` / `openai.gpt-5.4`) is NEVER paired +with a `converse`/`bedrock-runtime` path — bare ids are mantle ids. GPT-5.6, GPT-6, **GPT-5.5 and +GPT-5.4** MAY use `runtime_openai_cris` on `bedrock-runtime` with a CRIS-prefixed id — this is +not a violation (GPT-5.5 / GPT-5.4 runtime support: model cards read 2026-11-06, `us.`/`global.` +ids only, no runtime in-region form). Note: `runtime_openai_cris` for GPT-6 Astra is valid only via Responses/Chat Completions/Converse — Astra does not support `Invoke`, so a design that needs Guardrails or Application Inference Profiles (Converse-only features that also require Invoke-adjacent plumbing on some other models) must verify against Astra's model card @@ -201,9 +204,10 @@ rubric file (`ai.md`, `ai-azure-openai-to-bedrock.md`, or a `vendored/ai/*` ref) `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. - [ ] Bare proprietary GPT ids use Mantle; runtime Astra uses a supported `us.` / `global.` CRIS id and its caller-region gate. The recorded API and `model_change` match the design. -- [ ] No `bedrock_models[]` entry pairs **GPT-5.5 or GPT-5.4** with a `converse`/`bedrock-runtime` - migration path (those two models only — mantle-only, no CRIS). GPT-5.6 and GPT-6 models MAY - use `migration_path: "runtime_openai_cris"` on `bedrock-runtime` — this is NOT a violation. +- [ ] No `bedrock_models[]` entry pairs a **bare** proprietary GPT id (including bare + `openai.gpt-5.5` / `openai.gpt-5.4`) with a `converse`/`bedrock-runtime` migration path. + GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 MAY use `migration_path: "runtime_openai_cris"` on + `bedrock-runtime` with a `us.`/`in.`/`global.` CRIS id — this is NOT a violation. - [ ] If `ai_source` is `azure_openai` or `openai`: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-migration-guardrails.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-migration-guardrails.md index a819dc895..1ad1b0a09 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-migration-guardrails.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-migration-guardrails.md @@ -48,9 +48,9 @@ Inference on `bedrock-mantle` for these models is governed by **two per-model, p | Medium | Medium | Monitor 429s against **token** throughput, not request rate; enable prompt caching | | High | High | Enable prompt caching first (cached input is exempt from input TPM), then request a quota increase | -**The `bedrock-runtime` fallback exists only for GPT-5.6 and GPT-6 (Astra, Sol, Luna, 6.1 Sol, 6.1 Terra).** GPT-5.5 and GPT-5.4 are `bedrock-mantle` only and in-region only — for them, "switch to `bedrock-runtime`" requires moving to a different model (Bedrock-native or `gpt-oss`), a model change with its own eval cost. GPT-5.6 Sol/Terra/Luna and GPT-6 Astra/Sol/Luna/6.1 Sol/6.1 Terra DO have a `bedrock-runtime` path via CRIS inference profiles (GPT-5.6 `us.`/`in.`/`global.`, GPT-6 `us.`/`global.` prefixed ids; the model cards recommend runtime for new applications) — a legitimate endpoint option with its own quota family, and on Global CRIS it is also the cost-parity option. Scaling levers on the mantle path, in order: +**Every proprietary GPT model on Bedrock has a `bedrock-runtime` path via CRIS inference profiles.** GPT-5.6 Sol/Terra/Luna (`us.`/`in.`/`global.`), GPT-6 Astra/Sol/Luna/6.1 Sol/6.1 Terra (`us.`/`global.`), and GPT-5.5 / GPT-5.4 (`us.`/`global.`; per their model cards read 2026-11-06, [GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), [GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)) all list `bedrock-runtime` with Responses, Chat Completions and Converse; none has a runtime in-region form. The model cards recommend runtime for new applications. It is a legitimate endpoint option with its own quota family, and for GPT-5.6 / GPT-6 on Global CRIS it is also the cost-parity option (the GPT-5.5 / GPT-5.4 cards publish In-Region pricing only). A same-model endpoint switch is not a model change; only moving to a Bedrock-native model or `gpt-oss` carries the eval cost of a model change. Scaling levers on the mantle path, in order: -1. Prompt caching (GPT-5.6 and GPT-6 only) — 90% off cached input and exempt from the input-TPM quota +1. Prompt caching (GPT-5.6 and GPT-6 on mantle; GPT-5.5 / GPT-5.4 list implicit caching on `bedrock-runtime` Responses only) — 90% off cached input and exempt from the input-TPM quota 2. Exponential backoff with a bounded retry count (`max_retries` on the OpenAI SDK) 3. Spreading load across minutes and ramping request rate gradually rather than bursting 4. A Service Quotas increase for that model's input/output TPM in that region @@ -108,8 +108,8 @@ Claude models on Mantle have an additional **output TPM cap** that differs by mo - For Claude migrations at medium/high volume: the 2M output TPM cap on Claude 4.7+ is the binding constraint - For OpenAI proprietary GPT targets: this Claude cap does not apply. Use the GPT-5.x mantle quotas or - Astra-specific verification above. Astra / GPT-5.6 have runtime CRIS alternatives with their own - quotas; GPT-5.5/5.4 have no runtime path + Astra-specific verification above. Astra / GPT-5.6 / GPT-5.5 / GPT-5.4 all have runtime CRIS alternatives + with their own quotas (`us.`/`global.` prefixed ids; no runtime in-region form) - For `gpt-oss` targets: these do run on `bedrock-runtime`, so standard account TPM limits and the Converse-path mitigations apply - When output-heavy workloads (long JSON, tool outputs, multi-step reasoning) are detected, flag the relevant cap prominently; recommend `bedrock-runtime` for production **only** when the target model actually has a `bedrock-runtime` path diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index cfc5158ee..78fb426c0 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -42,7 +42,9 @@ Two consequences that invert the previous decision logic: 2. **The cost of a same-model move depends on the inference option — state it conditionally.** GPT-5.x In-Region and Geo CRIS are priced at OpenAI's _data residency_ tier, exactly 1.10x the standard list price, so those moves cost ~10% more. **GPT-5.6 on Global CRIS is priced at the standard list price — cost parity** — available only when - the workload has no data-residency constraint (GPT-5.5 / GPT-5.4 have no CRIS at all). See + the workload has no data-residency constraint. GPT-5.5 / GPT-5.4 also have `us.`/`global.` CRIS ids on + `bedrock-runtime`, but their model cards (read 2026-11-06) publish In-Region pricing only, so quote the ~10% + figure for them on every inference option until a Global rate is published. See `shared/openai-on-bedrock.md`. The non-cost case — AWS commitments, IAM/VPC/CloudTrail governance, residency, prompt caching, one vendor relationship — carries the in-region path; never present that path as free or neutral. For Astra, use its separately dated Bedrock rates and verify the source provider's rate before comparing. @@ -72,14 +74,15 @@ GPT-5.4, the target is **the same model on Bedrock**. | GPT-5.6 Sol | GPT-5.6 Sol | `openai.gpt-5.6-sol` / `us.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | | GPT-5.6 Terra | GPT-5.6 Terra | `openai.gpt-5.6-terra` / `us.` `in.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | | GPT-5.6 Luna | GPT-5.6 Luna | `openai.gpt-5.6-luna` / `us.` `in.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | -| GPT-5.5 | GPT-5.5 | `openai.gpt-5.5` (mantle only) | `strong_migrate` — same model, ~10% over OpenAI std | -| GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` (mantle only) | `strong_migrate` — same model, ~10% over OpenAI std | +| GPT-5.5 | GPT-5.5 | `openai.gpt-5.5` / `us.` `global.` prefixed | `strong_migrate` — same model, ~10% over OpenAI std | +| GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` / `us.` `global.` prefixed | `strong_migrate` — same model, ~10% over OpenAI std | Then apply the **region gate** below. Record `model_change: false` and the chosen path in `aws-design-ai.json` → `ai_architecture.code_migration`: `migration_path: "mantle_openai_chat"` for -Astra sources using Chat Completions, or `"mantle_openai_responses"` for a selected Mantle Responses -path. Use `migration_path: "runtime_openai_cris"` when a GPT-6 / GPT-5.6 target is served via -`bedrock-runtime` with a CRIS id (the model cards recommend runtime for new applications). +sources using Chat Completions on a model whose card lists it on mantle (GPT-6, GPT-5.5, GPT-5.4), or +`"mantle_openai_responses"` for a selected Mantle Responses path. Use `migration_path: "runtime_openai_cris"` +when a GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 target is served via `bedrock-runtime` with a CRIS id (the model +cards recommend runtime for new applications). Report the assessment honestly: `strong_migrate` here means "low-risk, well-supported move", not "cheaper". @@ -105,14 +108,16 @@ regions — offer it, stating the data-residency implication of cross-region rou Global routes anywhere and is the cost-parity option). Only a workload that requires strictly in-region processing in a region outside the mantle matrix falls through to Tier 1. -**GPT-5.5 / GPT-5.4:** hard gate. Mantle in-region only, no CRIS, no fallback. If the target region is not in the -matrix: +**GPT-5.5 / GPT-5.4:** same shape as GPT-5.6, with a narrower mantle matrix (GPT-5.5: us-east-1, us-east-2; +GPT-5.4: us-east-1, us-east-2, us-west-2, us-gov-west-1 — per their model cards, read 2026-11-06). If the target +region is in that matrix, either endpoint works. Otherwise the `bedrock-runtime` CRIS path (`us.openai.gpt-5.5` / +`global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`) reaches it, with the same data-residency +caveat as GPT-5.6 (Geo stays within the geography, Global routes anywhere). There is no runtime in-region form. +Only a workload that requires strictly in-region processing in a region outside the mantle matrix falls through: -1. Prefer offering a **region change** to a supported region, if no data-residency constraint pins the workload. - This preserves the zero-behavior-delta benefit, which is the whole point of Tier 0. -2. If the region is fixed, also offer **upgrading within the vendor to GPT-5.6** (which reaches the region via - CRIS) alongside the cross-family option — a generation change, but same vendor and prompt idioms. -3. Otherwise fall through to Tier 1 and say why explicitly — a region constraint, not a model judgment. +1. Prefer offering a **region change** to a supported mantle region, if no data-residency constraint pins the + workload. This preserves the zero-behavior-delta benefit, which is the whole point of Tier 0. +2. Otherwise fall through to Tier 1 and say why explicitly — a region constraint, not a model judgment. Record the outcome in `regional_warnings[]`. @@ -235,10 +240,11 @@ model on price. ### Open-weight gpt-oss -`openai.gpt-oss-120b` (0.15 / 0.60) and `openai.gpt-oss-20b` (0.07 / 0.30) remain available and, unlike the -proprietary GPT models, **do** support `bedrock-runtime` / Converse. They sit a capability class below the GPT-5.x -frontier tier. Recommend them when the user wants OpenAI-architecture models on the Bedrock-native runtime surface -(for Guardrails, invocation logging, or Converse-based tooling) rather than through mantle. +`openai.gpt-oss-120b` (0.15 / 0.60) and `openai.gpt-oss-20b` (0.07 / 0.30) remain available and support +`bedrock-runtime` / Converse with a bare model ID (no CRIS profile needed) and `Invoke`, which the proprietary GPT +models do not. They sit a capability class below the GPT-5.x frontier tier. Recommend them when the user wants +OpenAI-architecture models on the Bedrock-native runtime surface (for Guardrails, invocation logging, or +Converse-based tooling) at the lowest price rather than through mantle. --- @@ -284,7 +290,7 @@ rather than dismissing it. For a long-context (>272K) workload the gap is much w | Chat Completions | Preserve supported Chat when selected | Reshape only for a selected API change; verify the model and endpoint | | Function calling | Supported on GPT-5.x via mantle; Claude tools via Converse | Same-model keeps tool semantics identical | | Reasoning effort | `reasoning={"effort": ...}` on mantle | `none` / `low` / `medium` / `high` / `xhigh` / `max` | -| Prompt caching | GPT-5.6 only (90% off cached input) | Not listed for GPT-5.5 / 5.4; Claude has its own caching | +| Prompt caching | GPT-5.6 (90% off cached input); GPT-5.5 / 5.4 implicit caching on `bedrock-runtime` Responses only | GPT-5.5 / 5.4 cards (read 2026-11-06) list a cache-read rate and implicit caching on runtime, none on mantle; Claude has its own caching | | Streaming | Supported | Verify per surface | | Vision | GPT-5.x (image input) or Claude / Llama 4 | Same-model preserves behavior | | JSON mode | Claude (excellent), Nova Pro (good) | Most models via prompt | @@ -339,11 +345,12 @@ and closed to new customers as of July 30, 2026. ### When to prefer a Bedrock-native model over the mantle GPT path Choose Claude / Nova via Converse when the workload needs Knowledge Bases, intelligent prompt routing, structured -outputs, or application inference profiles — none of which the GPT path offers. Note what no longer forces a family -switch for GPT-5.6: its `bedrock-runtime` CRIS path provides cross-region inference, invocation logs, and -Guardrails (Converse API only; prompt caching is Responses-only on runtime). For GPT-5.5 / GPT-5.4, Guardrails, -logging, and cross-region reach still require a different model. The mantle GPT path is the smallest change; the -Bedrock-native path remains the most feature-complete. +outputs, or application inference profiles — none of which the mantle GPT path offers. Note what no longer forces +a family switch for GPT-5.6, GPT-5.5 and GPT-5.4: their `bedrock-runtime` CRIS paths provide cross-region +inference, invocation logs, Guardrails and application inference profiles (both Converse API only), and +structured outputs and implicit prompt caching (both Responses API only) — per the GPT-5.4 / GPT-5.5 model cards +read 2026-11-06. Knowledge Bases and intelligent prompt routing still require a Bedrock-native model. The mantle +GPT path is the smallest change; the Bedrock-native path remains the most feature-complete. ### High-spend tiering @@ -364,8 +371,8 @@ alternative, not against the source provider. cheaper and exempt from the input-TPM quota, which relieves the binding constraint on mantle. **Very high (>100M tokens/day):** tiering plus caching. Check TPM headroom per model per region early; there is no -RPM quota and no `bedrock-runtime` fallback for GPT models, so quota increases are the scaling lever. See -`shared/ai-migration-guardrails.md`. +RPM quota, and the `bedrock-runtime` CRIS path for GPT-5.6 / 5.5 / 5.4 / GPT-6 carries its own quotas rather than +adding headroom to mantle, so quota increases are the scaling lever. See `shared/ai-migration-guardrails.md`. --- diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md index 18266733c..a4206583c 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md @@ -133,8 +133,8 @@ See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommen | GPT-5.6 Sol | openai.gpt-5.6-sol | OpenAI | 4.40 | 22.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 8.80/33.00) | | GPT-5.6 Terra | openai.gpt-5.6-terra | OpenAI | 2.20 | 13.20 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 4.40/19.80) | | GPT-5.6 Luna | openai.gpt-5.6-luna | OpenAI | 0.22 | 1.32 | 272K | fast | active (mantle in-region + runtime CRIS; 1M tier 0.44/1.98) | -| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle only; no 1M tier) | -| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle only; no 1M tier) | +| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 11.00/49.50; card read 2026-11-06) | +| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 5.50/24.75; card read 2026-11-06) | | Gemma 4 31B | google.gemma-4-31b | Google | 0.14 | 0.40 | 256K | mid | active (Mantle only) | | Gemma 4 26B-A4B | google.gemma-4-26b-a4b | Google | 0.13 | 0.40 | 256K | efficient | active (Mantle only) | | Gemma 4 E2B | google.gemma-4-e2b | Google | 0.04 | 0.08 | 128K | budget | active (Mantle only) | @@ -353,17 +353,24 @@ Regions (mantle): us-east-1 / us-east-2 for all five; us-west-2 additionally car | GPT-5.5 | 5.50 | 33.00 | — | 0.55 | | GPT-5.4 | 2.75 | 16.50 | — | 0.275 | -#### Long context (1M) — GPT-5.6 only, 2.0x input / 1.5x output - -| Model | Input $/1M | Output $/1M | -| ------------- | ---------- | ----------- | -| GPT-5.6 Sol | 8.80 | 33.00 | -| GPT-5.6 Terra | 4.40 | 19.80 | -| GPT-5.6 Luna | 0.44 | 1.98 | - -A workload above 272K context must be priced at the long-context tier. GPT-5.5 and GPT-5.4 have no such tier, so -their usable window is 272K. Global CRIS is published at OpenAI's standard list price (cost parity, GPT-5.6 only — -see `openai-on-bedrock.md`). GovCloud GPT-5.4 is 3.30 / 19.80. +#### Long context (>272K input tokens) — GPT-5.6 at 2.0x input / 1.5x output; GPT-5.5 / GPT-5.4 per their model cards + +| Model | Input $/1M | Output $/1M | Cache read $/1M | +| ------------- | ---------- | ----------- | --------------- | +| GPT-5.6 Sol | 8.80 | 33.00 | — | +| GPT-5.6 Terra | 4.40 | 19.80 | — | +| GPT-5.6 Luna | 0.44 | 1.98 | — | +| GPT-5.5 | 11.00 | 49.50 | 1.10 | +| GPT-5.4 | 5.50 | 24.75 | 0.55 | + +A workload above 272K input tokens must be priced at the long-context tier, applied to all input and output tokens +of the request. GPT-5.5 and GPT-5.4 rows read from their model cards on 2026-11-06 +([GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), +[GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)); both cards state a +1,050,000-token context window (128,000 max output) and list `us.`/`global.` CRIS ids on `bedrock-runtime`, but +publish the In-Region inference option only. Global CRIS is published at OpenAI's standard list price (cost parity, +GPT-5.6 only — see `openai-on-bedrock.md`). GovCloud (US-West) GPT-5.4 short-context is 3.375 / 20.25 (cache read +0.3375) per the 2026-11-06 card; an earlier revision recorded 3.30 / 19.80. > **One source conflict.** These models are absent > from the AWS Price List API entirely: querying `AmazonBedrock` returns only `gpt-oss` and GPT OSS Safeguard, and @@ -373,7 +380,8 @@ see `openai-on-bedrock.md`). GovCloud GPT-5.4 is 3.30 / 19.80. **Prompt caching (GPT-5.6 only):** cached input read at a 90% discount, cache write at 1.25x uncached input, minimum 1,024-token prefix, up to 4 breakpoints, ≥30-minute retention. Cached input is exempt from the input-TPM quota. -GPT-5.5 and GPT-5.4 do not list prompt caching support. +GPT-5.5 and GPT-5.4 list only implicit prompt caching on `bedrock-runtime` Responses (automatic, no request +parameters; cache-read rate above, no cache-write rate) and none on mantle (model cards read 2026-11-06). #### Open-weight gpt-oss diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md index cdf8ced55..7720d4b58 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -190,12 +190,18 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` ## migration_path vocabulary `mantle_openai_responses` (an OpenAI-protocol source — direct or Azure-fronted — with the model -on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `converse` -(Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when a proprietary GPT -model must move off mantle for Guardrails / Knowledge Bases / logging / an unsupported region) · -`gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic -Bedrock SDK swap). A proprietary `openai.gpt-*` model ID is NEVER paired with a -`converse`/`bedrock-runtime` path — those models are mantle-only. +on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `mantle_openai_chat` +(same, preserving a Chat Completions source on a model whose card lists Chat Completions on +mantle — GPT-6, GPT-5.5, GPT-5.4) · `runtime_openai_cris` (same-model move served via +`bedrock-runtime` with a `us.`/`in.`/`global.` CRIS id, `model_change: false`; available for +GPT-6, GPT-5.6, GPT-5.5 and GPT-5.4 per their model cards) · `converse` (Bedrock-native +Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, +intelligent prompt routing, or another feature no same-model path serves) · `gpt-oss` +(OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK +swap). A **bare** proprietary `openai.gpt-*` model ID is NEVER paired with a +`converse`/`bedrock-runtime` path — bare ids are mantle ids; every proprietary GPT target on +`bedrock-runtime` carries a CRIS-prefixed id (there is no runtime in-region form for GPT-6, +GPT-5.6, GPT-5.5 or GPT-5.4). ## regional_validation @@ -255,8 +261,9 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` `price_comparison`. - [ ] `honest_assessment` overall is the weakest across `bedrock_models[]`; if `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. -- [ ] No `bedrock_models[]` entry pairs a proprietary `openai.gpt-*` model ID with a - `converse`/`bedrock-runtime` migration path (mantle-only). +- [ ] No `bedrock_models[]` entry pairs a **bare** proprietary `openai.gpt-*` model ID with a + `converse`/`bedrock-runtime` migration path (bare ids are mantle ids; a `bedrock-runtime` + target for GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 carries a `us.`/`in.`/`global.` CRIS id). - [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves to direct OpenAI: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. diff --git a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md index 299aae03f..1b5c2c4e5 100644 --- a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md @@ -225,8 +225,8 @@ _Skip when:_ the validator summary reports `models_extractable: true` — every **Same-model rows first.** GPT-6 Astra / GPT-5.6 / 5.5 / 5.4 sources map to themselves. Design applies the model-specific API and region gate; Clarify records the baseline only. -Astra Standard mantle supports `us-east-1` and `us-west-2`; runtime uses US Geo / Global CRIS. GPT-5.6 also has runtime CRIS; -GPT-5.5 / 5.4 remain mantle-only. GPT-5.x in-region costs about 10% above OpenAI standard, +Astra Standard mantle supports `us-east-1` and `us-west-2`; runtime uses US Geo / Global CRIS. GPT-5.6, GPT-5.5 +and GPT-5.4 also have runtime CRIS (`us.` / `global.`). GPT-5.x in-region costs about 10% above OpenAI standard, while GPT-5.6 Global CRIS uses the standard rate. For Astra, compare its separately dated inference/context tier against the source price before claiming savings or parity. For sources without a Bedrock equivalent, present a same-vendor upgrade and a cross-family option. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md index 0890b0e15..8634efdc5 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md @@ -204,8 +204,8 @@ _Skip when:_ `models[].model_id` is populated in `ai-workload-profile.json` **wi **Same-model rows first.** GPT-6 Astra / GPT-5.6 / 5.5 / 5.4 sources map to themselves. Apply the model-specific API and region gate in `references/shared/openai-on-bedrock.md`. -Astra Standard mantle supports `us-east-1` and `us-west-2`; runtime uses US Geo / Global CRIS. GPT-5.6 also has runtime CRIS; -GPT-5.5 / 5.4 remain mantle-only. GPT-5.x in-region costs about 10% above OpenAI standard, +Astra Standard mantle supports `us-east-1` and `us-west-2`; runtime uses US Geo / Global CRIS. GPT-5.6, GPT-5.5 +and GPT-5.4 also have runtime CRIS (`us.` / `global.`). GPT-5.x in-region costs about 10% above OpenAI standard, while GPT-5.6 Global CRIS uses the standard rate. For Astra, compare its separately dated inference/context tier against the source price before claiming savings or parity. For sources without a Bedrock equivalent, present a same-vendor upgrade and a cross-family option. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index 944d109dd..a744a6faf 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -238,7 +238,9 @@ data-residency-tier premium, while supported Global CRIS uses standard-rate pari Report the actual option; do not downgrade solely for the modest verified premium. **GPT-5.6/GPT-6 only:** above 272K context, apply 2.0x input and 1.5x output rates only when the selected model and inference option support that tier; the resulting gap may change -the assessment. GPT-5.5/5.4 have no 1M tier. **Opus 5.5:** use the selected region/profile's +the assessment. GPT-5.5/5.4 have their own long-context (>272K) rates on their model cards +(read 2026-11-06; see `openai-on-bedrock.md` § Pricing) — use those, not the GPT-5.6 +multiplier. **Opus 5.5:** use the selected region/profile's flat rates from `references/vendored/ai/bedrock-pricing-cache.md` throughout the full 1M context; do not apply the GPT multiplier. Verify other models' tiers individually. @@ -407,23 +409,25 @@ Astra Chat solely because it is proprietary GPT. Record `model_change: false` on the exact source model is retained; a Pro-to-Astra upgrade is a model change requiring eval. GPT-5.x retains its separately dated API evidence and any selected Chat-to-Responses reshape. -**A Chat Completions source is a supported near-drop-in for GPT-6 and GPT-5.6 — not a mandatory reshape.** All five GPT-6 models and the GPT-5.6 family list Chat Completions as a supported API on both mantle and runtime CRIS (per `openai-on-bedrock.md` § Access Paths); if `integration` evidence shows `chat.completions.create` for one of these models, the design may keep `chat.completions.create`/`choices[0].message` parsing — reshape to `responses.create` is optional, only for a Responses-only feature (runtime explicit prompt caching, reasoning-item round-trips) or a demonstrated constraint. **GPT-5.5 and GPT-5.4 are the exception**: Chat Completions support is unconfirmed on their model cards (Responses on mantle is the only verified surface), so a Chat Completions source on either model still needs a planned reshape, probed against the target account before committing. +**A Chat Completions source is a supported near-drop-in for GPT-6 and GPT-5.6 — not a mandatory reshape.** All five GPT-6 models and the GPT-5.6 family list Chat Completions as a supported API on both mantle and runtime CRIS (per `openai-on-bedrock.md` § Access Paths); if `integration` evidence shows `chat.completions.create` for one of these models, the design may keep `chat.completions.create`/`choices[0].message` parsing — reshape to `responses.create` is optional, only for a Responses-only feature (runtime explicit prompt caching, reasoning-item round-trips) or a demonstrated constraint. **GPT-5.5 and GPT-5.4 follow the same rule**: their model cards (read 2026-11-06) list Chat Completions as supported on both `bedrock-mantle` and `bedrock-runtime`, so a Chat Completions source on either model may keep Chat Completions; the reshape is optional for them too. A planned reshape is required only when the selected model's card leaves Chat Completions unchecked on the selected endpoint (check the per-model path entries in the canonical catalog). Probe the selected API against the target account before committing, as for any model. **Terra caching depends on the endpoint and mode.** On runtime, implicit caching supports Responses, Chat Completions, InvokeModel, and Converse. It does not require a Chat client reshape and can be combined with Converse Guardrails. Runtime explicit caching requires Responses. On Mantle, both implicit and explicit caching are Responses-only. -**No Converse fallback for GPT-5.5/GPT-5.4 — GPT-5.6 and GPT-6 differ.** GPT-5.5 and GPT-5.4 are -`bedrock-mantle` only and in-region only — there is no `bedrock-runtime` path and no cross-region -inference profile for these two models. GPT-5.6 Sol/Terra/Luna and all five GPT-6 models (Astra, -Sol, Luna, 6.1 Sol, 6.1 Terra) DO have a `bedrock-runtime` CRIS path (`us.`/`global.` prefixed IDs, per -`openai-on-bedrock.md` § Access Paths) — for these models, a region or feature need (Guardrails, -Knowledge Bases, invocation logging, or a region outside the mantle matrix) is satisfied by -**switching endpoint, not model**: record `migration_path: "runtime_openai_cris"` with -`model_change: false`. Reserve `migration_path: "converse"` with `model_change: true` for -GPT-5.5/GPT-5.4 only, or when the GPT-6/5.6 CRIS path itself cannot serve the need (e.g. a feature -CRIS does not support — see the runtime-side feature splits in `openai-on-bedrock.md`). Note: +**Every proprietary GPT model on Bedrock has a `bedrock-runtime` CRIS path — including GPT-5.5 and +GPT-5.4.** GPT-5.6 Sol/Terra/Luna, all five GPT-6 models (Astra, Sol, Luna, 6.1 Sol, 6.1 Terra), and +GPT-5.5 / GPT-5.4 (model cards read 2026-11-06: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, +`us.openai.gpt-5.4` / `global.openai.gpt-5.4`; Responses, Chat Completions and Converse on runtime) +have a `bedrock-runtime` CRIS path (`us.`/`global.` prefixed IDs, per `openai-on-bedrock.md` +§ Access Paths). No proprietary GPT model has a runtime in-region form. For all of them, a region +or feature need (Guardrails, invocation logging, application inference profiles, or a region +outside the mantle matrix) is satisfied by **switching endpoint, not model**: record +`migration_path: "runtime_openai_cris"` with `model_change: false`. Reserve +`migration_path: "converse"` with `model_change: true` for a need the same-model CRIS path itself +cannot serve (Knowledge Bases, intelligent prompt routing, or a feature CRIS does not support — +see the runtime-side feature splits in `openai-on-bedrock.md`). Note: `runtime_openai_cris` for GPT-6 Astra is valid only via Responses/Chat Completions/Converse — Astra does not support `Invoke`, so a design that needs Guardrails or Application Inference Profiles (Converse-only features that also require Invoke-adjacent plumbing on some other models) @@ -511,9 +515,10 @@ Write `aws-design-ai.json` to `$MIGRATION_DIR/`. - [ ] `honest_assessment` logic is consistent (weakest model drives overall) - [ ] If `ai_source` includes `"openai"`: every model that `shared/openai-on-bedrock.md` lists as available on Bedrock, and whose target region carries it, is mapped to **itself** with `model_change: false` — not to a Claude/Nova substitute - [ ] If `honest_assessment == "recommend_stay"`: `honest_assessment_reason` names a non-cost blocker (cost parity alone is never sufficient for an OpenAI source) -- [ ] No `bedrock_models[]` entry pairs **GPT-5.5 or GPT-5.4** with a Converse/`bedrock-runtime` - migration path (those two models only — mantle-only, no CRIS). GPT-5.6 and GPT-6 models MAY - use `migration_path: "runtime_openai_cris"` on `bedrock-runtime` — this is NOT a violation. +- [ ] No `bedrock_models[]` entry pairs a **bare** proprietary GPT id (including bare + `openai.gpt-5.5` / `openai.gpt-5.4`) with a Converse/`bedrock-runtime` migration path. + GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 MAY use `migration_path: "runtime_openai_cris"` on + `bedrock-runtime` with a `us.`/`in.`/`global.` CRIS id — this is NOT a violation. - [ ] `regional_warnings` is present (empty array `[]` if no issues; populated if any service unavailable in target region) - [ ] `multi_model_warnings` is present (empty array `[]` if single model or no coordination issues) - [ ] If `agentic_profile.is_agentic == true`: `agentic_design` object is present with `migration_approach` matching `preferences.json` diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-preview.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-preview.md index dd54a54e9..25f024e7a 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-preview.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-preview.md @@ -104,8 +104,8 @@ See `references/shared/openai-on-bedrock.md` for the model-specific pricing rule For the GPT-5.x rows, the mantle endpoint is in-region only (us-east-1, us-east-2; us-west-2 additionally for Terra, Luna, and GPT-5.4; AWS GovCloud us-gov-west-1 / us-gov-east-1 for Terra and Luna, us-gov-west-1 also for -GPT-5.4). GPT-5.6 additionally reaches most commercial regions via `bedrock-runtime` CRIS ids; GPT-5.5 / GPT-5.4 -have no CRIS. +GPT-5.4). GPT-5.6, GPT-5.5 and GPT-5.4 additionally reach most commercial regions via `bedrock-runtime` CRIS ids +(`us.` / `global.`; GPT-5.5 / GPT-5.4 per their model cards read 2026-11-06). For the GPT-6 rows, mantle is us-east-1 only, except GPT-6 Astra which is also available in us-west-2. All five GPT-6 models additionally reach a broad set of commercial regions via `bedrock-runtime` CRIS (`us.` / `global.` diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md index 2571e0def..5fbeb17bc 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md @@ -32,7 +32,7 @@ Check `preferences.json` → `ai_constraints.ai_framework` to determine the migr - `mantle` / `mantle_openai_responses`: point the client at `https://bedrock-mantle.{region}.api.aws/openai/v1` — note the `openai/v1` segment for proprietary GPT models; a bare `/v1` 404s. Swap the credential for a Bedrock API key or the auto-refreshing token provider — **not** the existing OpenAI key. Update the model string to the bare Bedrock model ID (`openai.gpt-*` for a same-model move). Grant the `bedrock-mantle:*` actions; `bedrock:InvokeModel` does not authorize these models. - `runtime_openai_cris` (GPT-5.6/GPT-6 via `bedrock-runtime` CRIS): point the client at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` instead — **not** the mantle URL. Update the model string to the `us.`/`in.`/`global.`-prefixed CRIS inference-profile ID, not the bare ID. Grant standard `bedrock:InvokeModel*` actions against the CRIS inference-profile ARN — **not** `bedrock-mantle:*`. For a GPT-6 CRIS id, also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` alongside the inference-profile ARN (per the GPT-6 model cards). Check `aws-design-ai.json` → `ai_architecture.bedrock_models[].required_client_api`: if non-Responses, emit that surface (e.g. Converse) instead of defaulting to `responses.create` — absent/`responses` keeps the Responses surface for a region-only move. -- **Config-only if the source already calls `responses.create`.** A Chat Completions source for GPT-6 or GPT-5.6 may keep `chat.completions.create`/`choices[0].message` parsing unchanged — Chat Completions is supported on both mantle and runtime CRIS for these models, so reshape to Responses is optional (only for a Responses-only feature or demonstrated constraint). For GPT-5.5/GPT-5.4, Chat Completions support is unconfirmed, so still plan a reshape to Responses and probe the target account before committing. +- **Config-only if the source already calls `responses.create`.** A Chat Completions source for GPT-6 or GPT-5.6 may keep `chat.completions.create`/`choices[0].message` parsing unchanged — Chat Completions is supported on both mantle and runtime CRIS for these models, so reshape to Responses is optional (only for a Responses-only feature or demonstrated constraint). GPT-5.5/GPT-5.4 follow the same rule — their model cards (read 2026-11-06) list Chat Completions on both mantle and runtime, so the reshape is optional for them too; probe the selected API against the target account before committing, as for any model. - Test in staging, validate responses Match the `migration_path` prefix, not the exact string: Design writes the more specific `mantle_openai_responses` for a same-model OpenAI migration, and an equality test against `mantle` would misroute those runs into the adapter path below. `runtime_openai_cris` (GPT-5.6/GPT-6 served via `bedrock-runtime` CRIS instead of `bedrock-mantle`) is matched explicitly alongside the `mantle*` prefix — it does not start with `mantle`, so it needs its own check to avoid falling through to the adapter path below. @@ -55,7 +55,7 @@ Based on `ai-workload-profile.json` → `integration.pattern` and `integration.l | Source SDK | Target | Key Change | | ----------------------- | --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| OpenAI SDK (same model) | Mantle Responses API | Base URL → `.../openai/v1`, Bedrock token/API key (not an OpenAI key), `openai.gpt-*` model ID. Config-only **if** already on `responses.create`; a Chat Completions source may keep Chat Completions for GPT-6/GPT-5.6 (reshape to Responses optional); GPT-5.5/GPT-5.4 still need a reshape. | +| OpenAI SDK (same model) | Mantle Responses API | Base URL → `.../openai/v1`, Bedrock token/API key (not an OpenAI key), `openai.gpt-*` model ID. Config-only **if** already on `responses.create`; a Chat Completions source may keep Chat Completions for GPT-6/GPT-5.6/GPT-5.5/GPT-5.4 (reshape to Responses optional; plan it only when the selected model's card leaves Chat Completions unchecked on the selected endpoint). | | Vertex AI (Python) | boto3 Bedrock Converse API | `GenerativeModel.generate_content()` → `bedrock.converse()` | | Vertex AI (JS) | @aws-sdk/client-bedrock-runtime | `model.generateContent()` → `client.send(new ConverseCommand())` | | Vertex AI (Go) | aws-sdk-go-v2 bedrockruntime | `aiplatform` → `bedrockruntime.Converse()` | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md index eb6969816..492bced1e 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md @@ -2,6 +2,7 @@ **Last verified:** 2026-10-10 **GPT-6.1 Terra verified:** 2026-10-27 +**GPT-5.5 / GPT-5.4 re-verified:** 2026-11-06 against their model cards ([GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), [GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)) — both now list `bedrock-runtime` (Responses, Chat Completions, Converse via `us.`/`global.` CRIS ids), Chat Completions on mantle, a 1.05M context window and a long-context price tier **Sources:** [OpenAI model cards](https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html) (per-model cards linked below), [GPT-5.6 launch post](https://aws.amazon.com/blogs/machine-learning/get-started-with-openai-gpt-5-6-sol-terra-and-luna-on-amazon-bedrock/), [GPT-5.6 GA announcement](https://aws.amazon.com/about-aws/whats-new/2026/07/openai-gpt-sol-terra/), @@ -32,8 +33,8 @@ risk) all defer to it. Do not restate model IDs, regions, or endpoint paths else | GPT-5.6 Sol | `openai.gpt-5.6-sol` | Jul 13, 2026 | 1M | Active | [card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html) | | GPT-5.6 Terra | `openai.gpt-5.6-terra` | Jul 13, 2026 | 1M | Active | [card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html) | | GPT-5.6 Luna | `openai.gpt-5.6-luna` | Jul 13, 2026 | 1M | Active | [card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html) | -| GPT-5.5 | `openai.gpt-5.5` | Jun 1, 2026 | 272K | Active | [card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html) | -| GPT-5.4 | `openai.gpt-5.4` | Jun 1, 2026 | 272K | Active | [card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html) | +| GPT-5.5 | `openai.gpt-5.5` | Jun 1, 2026 | 1.05M | Active | [card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html) | +| GPT-5.4 | `openai.gpt-5.4` | Jun 1, 2026 | 1.05M | Active | [card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html) | | gpt-oss-120b | `openai.gpt-oss-120b` | Aug 5, 2025 | 128K | Active | open-weight; also on `bedrock-runtime` as `openai.gpt-oss-120b-1:0` | | gpt-oss-20b | `openai.gpt-oss-20b` | Aug 5, 2025 | 128K | Active | open-weight; also on `bedrock-runtime` as `openai.gpt-oss-20b-1:0` | | GPT OSS Safeguard | `openai.gpt-oss-safeguard-120b` / `-20b` | — | — | Active | content-moderation / guardrail enforcement, not general chat | @@ -50,8 +51,10 @@ GPT-6 Astra is the newest, most capable OpenAI model in this catalog. Sol remain lower-cost reasoning alternative; Astra does not make existing GPT-5.x sources require a model upgrade. > **Context-window conflict (resolved):** the GPT-5.6 launch blog states 272K for all three variants; all three -> model cards state 1M. **The model cards are authoritative** — use 1M for GPT-5.6. GPT-5.5 and GPT-5.4 are 272K on -> both sources. Re-check on refresh; if AWS corrects the blog, the cards still win. +> model cards state 1M. **The model cards are authoritative** — use 1M for GPT-5.6. The GPT-5.5 and GPT-5.4 cards +> (read 2026-11-06) state 1,050,000 tokens shared between input and output with 128,000 max output; the 272K figure +> recorded earlier is now only their short-/long-context pricing breakpoint. Re-check on refresh; if AWS corrects +> the blog, the cards still win. **Not on Bedrock (as of this refresh):** GPT-4o, GPT-4.1, GPT-4 / GPT-4 Turbo, GPT-3.5 Turbo, the o-series (o1/o3/o4-mini), GPT-5 / GPT-5.1 / GPT-5.2, Codex (unverified — see the Codex note below), and the `*-Pro` variants @@ -116,8 +119,12 @@ input and text output. The launch announcement rounds the input context to 1M; u exact limits for capacity checks. Browser/computer-use capability does not establish parity with OpenAI-hosted tools; probe the application's specific tool flow. -**GPT-5.5 and GPT-5.4 are `bedrock-mantle`-only and in-region only.** Their model cards list a single -Programmatic Access row (`bedrock-mantle`, Geo/Global "Not supported") and In-Region pricing only. +**GPT-5.5 and GPT-5.4 have TWO endpoints** (re-verified 2026-11-06 against their model cards; earlier revisions of +this file recorded them as `bedrock-mantle`-only). Both Programmatic Access tables now carry a `bedrock-runtime` +row (In-Region "Not supported"; Geo `us.openai.gpt-5.5` / `us.openai.gpt-5.4`; Global `global.openai.gpt-5.5` / +`global.openai.gpt-5.4`) and a `bedrock-mantle` row (bare id, in-region only). Supported APIs: `Responses`, +`Chat Completions` and `Converse` on `bedrock-runtime`; `Responses` and `Chat Completions` on `bedrock-mantle`; +`Messages` and `Invoke` on neither. The pricing tables still publish the In-Region inference option only. **GPT-5.6 Sol / Terra / Luna have TWO endpoints** (verified 2026-08-21 — this changed after this file's original 2026-08-10 verification; the Refresh Checklist predicted it): @@ -136,13 +143,16 @@ Constraints that still break naive assumptions: mantle path explicitly, and the runtime example gives `bedrock-runtime.{region}.amazonaws.com/openai/v1`. 2. **The endpoints take different model-id forms.** Mantle takes the bare `openai.gpt-5.6-*` id and has no CRIS; `bedrock-runtime` takes ONLY a CRIS inference-profile id (`us.` / `in.` / `global.` prefixed) and has no - in-region form. `bedrock:ListInferenceProfiles` returns the 5.6 CRIS profiles; it never returns the bare mantle - ids, and it returns nothing for GPT-5.5 / GPT-5.4. + in-region form. `bedrock:ListInferenceProfiles` returns the CRIS profiles (5.6, and `us.`/`global.` for + GPT-5.5 / GPT-5.4 per their cards); it never returns the bare mantle ids. 3. **API surfaces differ by endpoint.** The 5.6 cards list `Responses`, `Chat Completions`, `Invoke`, and `Converse` as supported APIs, with runtime-side feature splits: **Guardrails are Converse-only; prompt caching is Responses-only on runtime**; server-side tool use, structured outputs, and application inference profiles are NOT - supported on runtime (server-side tool calling IS supported on mantle). For GPT-5.5 / GPT-5.4 treat Responses on - mantle as the only verified surface. + supported on runtime (server-side tool calling IS supported on mantle). The GPT-5.5 / GPT-5.4 cards (read + 2026-11-06) list `Responses`, `Chat Completions` and `Converse` on runtime (no `Invoke`) and `Responses` and + `Chat Completions` on mantle; on runtime they list Guardrails and application inference profiles as + Converse-only, and implicit prompt caching and structured outputs as Responses-only. Chat Completions is a + documented surface for GPT-5.5 / GPT-5.4 on both endpoints — probe it in the target account like any other. ### GPT-6 family: Ultrafast tier @@ -193,8 +203,10 @@ that expires within 12 hours and is not refreshed. Do not recommend it for produ **IAM:** on the mantle path, the managed policy `AmazonBedrockMantleInferenceAccess` grants what inference needs, including `bedrock-mantle:CreateInference` and `bedrock-mantle:CallWithBearerToken` — `bedrock:InvokeModel` does not -authorize mantle calls. On the GPT-5.6 `bedrock-runtime` path the usual `bedrock:InvokeModel*` against the CRIS -inference-profile ARN applies, as for any other runtime model. +authorize mantle calls. On the GPT-5.6 / GPT-5.5 / GPT-5.4 `bedrock-runtime` path the usual `bedrock:InvokeModel*` +against the CRIS inference-profile ARN applies, as for any other runtime model; the GPT-5.5 / GPT-5.4 cards add +that the identity also needs `bedrock:InvokeModel` on the account's default project +(`arn:aws:bedrock:{region}:{account-id}:project/default`). **Reasoning effort:** the five GPT-5.x models accept `none`, `low`, `medium`, `high`, `xhigh`, `max`. Astra's card does not enumerate these values; verify its accepted settings rather than inheriting the GPT-5.x list. Because these models reason @@ -205,7 +217,8 @@ request for multi-turn and tool-calling flows. ## Regional Availability — Endpoint-Aware -**The mantle in-region matrix** (the only reach for GPT-5.5 / GPT-5.4, and the in-region option for GPT-6 Astra / GPT-5.6): +**The mantle in-region matrix** (the in-region option for every proprietary GPT model; GPT-5.5 / GPT-5.4 rows +re-verified 2026-11-06): | Model | us-east-1 | us-east-2 | us-west-2 | us-gov-west-1 | us-gov-east-1 | | ------------- | --------- | --------- | --------- | ------------- | ------------- | @@ -235,8 +248,16 @@ any GPT-6 model, it just means using the runtime endpoint with a CRIS id. **GPT-5.6 additionally reaches most commercial regions via `bedrock-runtime` CRIS** (Geo `us.` / `in.`, Global `global.` inference profiles; the Sol card's runtime footprint spans 30+ regions). So a region outside the mantle matrix does NOT make the same-model path unavailable for GPT-5.6 — it means using the runtime endpoint with a CRIS -id, with the data-residency implications of cross-region routing. For GPT-5.5 / GPT-5.4 the mantle matrix is a hard -gate: no CRIS, no fallback. +id, with the data-residency implications of cross-region routing. + +**GPT-5.5 / GPT-5.4 runtime availability** (model cards read 2026-11-06): US Geo CRIS (`us.openai.gpt-5.5` / +`us.openai.gpt-5.4`) can be called from `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `ca-central-1` and +`ca-west-1`. Global CRIS (`global.openai.gpt-5.5` / `global.openai.gpt-5.4`) is listed in those six plus +`mx-central-1`, every EU region (`eu-central-1/2`, `eu-north-1`, `eu-south-1/2`, `eu-west-1/2/3`), `ap-east-2`, +`ap-northeast-1/2/3`, `ap-south-1/2`, `ap-southeast-1` through `-7`, `il-central-1`, `me-central-1`, `me-south-1`, +`af-south-1` and `sa-east-1` (33 regions). No runtime in-region form, no EU/India Geo profile, and no GovCloud +runtime row. So the mantle matrix is no longer a hard gate for GPT-5.5 / GPT-5.4; only a strictly in-region +requirement outside the mantle matrix blocks the same-model path. Verify current footprints per model card / `aws___get_regional_availability` (AWS MCP Server) — the CRIS lists move faster than this file. @@ -252,7 +273,8 @@ inference-option dimension:** workload has no data-residency constraint. So the honest cost statement is conditional, not flat: a same-model GPT-5.6 move on Global CRIS is -**cost-neutral**; the same move in-region or Geo (and any GPT-5.5 / GPT-5.4 move) is **~10% more expensive**. The +**cost-neutral**; the same move in-region or Geo is **~10% more expensive**. The GPT-5.5 / GPT-5.4 cards publish +In-Region pricing only (no Geo/Global rows as of 2026-11-06), so quote ~10% for them on every inference option. The same Global-CRIS-is-cost-neutral / in-Region-and-Geo-is-~10%-more pattern applies to the GPT-6 family below. Never state either number without stating the inference option it belongs to. @@ -307,12 +329,15 @@ provider rate before claiming savings or parity for the selected context tier. A workload above 272K context must be priced at the long-context tier. -### GPT-5.5 / GPT-5.4 — In-Region only (no CRIS, no long-context tier; usable window 272K) +### GPT-5.5 / GPT-5.4 — In-Region pricing only; short- and long-context tiers (model cards read 2026-11-06) + +The cards publish only the In-Region inference option (the `us.` / `global.` runtime CRIS ids exist but carry no +separate price row). Long-context rates apply to all input and output tokens of a request above 272K input tokens. -| Model | In-Region (in · out) | Cache read | Notes | -| ------- | -------------------- | ---------- | -------------------------------- | -| GPT-5.5 | 5.50 · 33.00 | 0.55 | no cache-write rate published | -| GPT-5.4 | 2.75 · 16.50 | 0.275 | GovCloud (US-West): 3.30 · 19.80 | +| Model | Short context ≤272K (in · cache read · out) | Long context >272K (in · cache read · out) | Notes | +| ------- | ------------------------------------------- | ------------------------------------------ | ---------------------------------------- | +| GPT-5.5 | 5.50 · 0.55 · 33.00 | 11.00 · 1.10 · 49.50 | no cache-write rate published | +| GPT-5.4 | 2.75 · 0.275 · 16.50 | 5.50 · 0.55 · 24.75 | GovCloud (US-West) short: 3.375 · 0.3375 · 20.25 | > **The Luna "blog discrepancy" resolved differently than first recorded.** The AWS News Blog's 0.20 / 1.20 is not > an error — it is the **Global CRIS** rate, now published on the Luna card. An earlier revision of this file said @@ -329,8 +354,10 @@ Its card does not establish the GPT-5.6 minimum-prefix, breakpoint, or quota-exe Listed as a supported feature on the GPT-5.6 Sol/Terra/Luna model cards and on all five GPT-6 model cards (Astra, Sol, Luna, 6.1 Sol, 6.1 Terra — confirmed by the cache-write/cache-read columns in the GPT-6 pricing tables above). The -GPT-5.5 and GPT-5.4 cards list client-side tool calling in that slot instead and do **not** list prompt caching. -Do not assume caching on 5.5/5.4. +GPT-5.5 and GPT-5.4 cards (read 2026-11-06) list **implicit prompt caching on `bedrock-runtime` Responses only** +(automatic, no request parameters) with a cache-read rate and no cache-write rate; explicit caching is unsupported +and the mantle feature list carries client-side tool calling in that slot instead. Do not assume caching on +5.5/5.4 mantle, and do not inherit the GPT-5.6 prefix/breakpoint rules below for them. | Property | Value | | ----------------- | ------------------------------------------------------------ | @@ -359,10 +386,10 @@ This corrects two claims that were previously applied to all Mantle traffic in t - There is **no shared 10,000 RPM account limit** governing these models — the quota dimension is TPM, per model, per region. -- "Switch to `bedrock-runtime`" is **not a throughput remedy for GPT-5.5 / GPT-5.4** — they have no - `bedrock-runtime` path at all. For GPT-5.6 the runtime path DOES exist (CRIS only; see Access Paths above), so - moving there is a legitimate option — but treat it as an endpoint/architecture choice with its own quota family - and residency implications, not a free throughput escape hatch. +- "Switch to `bedrock-runtime`" is **not a free throughput escape hatch**. For GPT-5.6, GPT-5.5 and GPT-5.4 the + runtime path exists (CRIS only; see Access Paths above — GPT-5.5 / GPT-5.4 re-verified 2026-11-06), so moving + there is a legitimate option — but treat it as an endpoint/architecture choice with its own quota family and + residency implications. The supported mitigations are: exponential backoff with a bounded retry count (`max_retries` on the OpenAI SDK), spreading load across minutes rather than bursting, ramping request rate gradually, and prompt caching (cached input @@ -409,8 +436,8 @@ This model family is moving fast (two GA waves and a repricing inside 10 weeks). 1. Re-read the [OpenAI model card index](https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html) for models added or removed, and each per-model card for lifecycle state and EOL date. -2. Recheck the region matrix AND the GPT-5.6 CRIS footprints — for 5.5/5.4 a region change is migration-blocking; - for 5.6 the runtime/CRIS path usually covers the region instead. +2. Recheck the region matrix AND the CRIS footprints for every proprietary GPT model — the runtime/CRIS path + usually covers a region outside the mantle matrix; only a strictly in-region requirement there is blocking. 3. Recheck rates on the Bedrock pricing page OpenAI tab, and resolve any row still marked _unverified_. 4. Recheck whether the Price List API has gained GPT-5.x coverage; if it has, drop the caveat above and let `estimate-ai.md` price these models from the MCP. @@ -424,3 +451,9 @@ This model family is moving fast (two GA waves and a repricing inside 10 weeks). 2026), and GPT-6.1 Sol (GA Sep 29, 2026) are all confirmed on Bedrock with their own model cards and pricing tables. New discovery this refresh: a 6x-Standard **Ultrafast** service tier on Astra and 6.1 Sol only (Sol and Luna remain Standard-only) — the first new tier shape since this file started tracking the family. +9. **2026-11-06 refresh — GPT-5.5 / GPT-5.4 re-read.** Item 5 fired again: both cards now list `bedrock-runtime` + (Responses, Chat Completions, Converse via `us.` / `global.` CRIS ids; no in-region runtime form), Chat + Completions on `bedrock-mantle`, a 1,050,000-token context window with 128,000 max output, a long-context + (>272K) price tier, implicit prompt caching on runtime Responses, and a GovCloud (US-West) GPT-5.4 rate of + 3.375 · 20.25. Earlier "mantle-only / no CRIS / 272K / no long-context tier" claims for these two models were + stale and are corrected throughout this file. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-migration-guardrails.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-migration-guardrails.md index a819dc895..1ad1b0a09 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-migration-guardrails.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-migration-guardrails.md @@ -48,9 +48,9 @@ Inference on `bedrock-mantle` for these models is governed by **two per-model, p | Medium | Medium | Monitor 429s against **token** throughput, not request rate; enable prompt caching | | High | High | Enable prompt caching first (cached input is exempt from input TPM), then request a quota increase | -**The `bedrock-runtime` fallback exists only for GPT-5.6 and GPT-6 (Astra, Sol, Luna, 6.1 Sol, 6.1 Terra).** GPT-5.5 and GPT-5.4 are `bedrock-mantle` only and in-region only — for them, "switch to `bedrock-runtime`" requires moving to a different model (Bedrock-native or `gpt-oss`), a model change with its own eval cost. GPT-5.6 Sol/Terra/Luna and GPT-6 Astra/Sol/Luna/6.1 Sol/6.1 Terra DO have a `bedrock-runtime` path via CRIS inference profiles (GPT-5.6 `us.`/`in.`/`global.`, GPT-6 `us.`/`global.` prefixed ids; the model cards recommend runtime for new applications) — a legitimate endpoint option with its own quota family, and on Global CRIS it is also the cost-parity option. Scaling levers on the mantle path, in order: +**Every proprietary GPT model on Bedrock has a `bedrock-runtime` path via CRIS inference profiles.** GPT-5.6 Sol/Terra/Luna (`us.`/`in.`/`global.`), GPT-6 Astra/Sol/Luna/6.1 Sol/6.1 Terra (`us.`/`global.`), and GPT-5.5 / GPT-5.4 (`us.`/`global.`; per their model cards read 2026-11-06, [GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), [GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)) all list `bedrock-runtime` with Responses, Chat Completions and Converse; none has a runtime in-region form. The model cards recommend runtime for new applications. It is a legitimate endpoint option with its own quota family, and for GPT-5.6 / GPT-6 on Global CRIS it is also the cost-parity option (the GPT-5.5 / GPT-5.4 cards publish In-Region pricing only). A same-model endpoint switch is not a model change; only moving to a Bedrock-native model or `gpt-oss` carries the eval cost of a model change. Scaling levers on the mantle path, in order: -1. Prompt caching (GPT-5.6 and GPT-6 only) — 90% off cached input and exempt from the input-TPM quota +1. Prompt caching (GPT-5.6 and GPT-6 on mantle; GPT-5.5 / GPT-5.4 list implicit caching on `bedrock-runtime` Responses only) — 90% off cached input and exempt from the input-TPM quota 2. Exponential backoff with a bounded retry count (`max_retries` on the OpenAI SDK) 3. Spreading load across minutes and ramping request rate gradually rather than bursting 4. A Service Quotas increase for that model's input/output TPM in that region @@ -108,8 +108,8 @@ Claude models on Mantle have an additional **output TPM cap** that differs by mo - For Claude migrations at medium/high volume: the 2M output TPM cap on Claude 4.7+ is the binding constraint - For OpenAI proprietary GPT targets: this Claude cap does not apply. Use the GPT-5.x mantle quotas or - Astra-specific verification above. Astra / GPT-5.6 have runtime CRIS alternatives with their own - quotas; GPT-5.5/5.4 have no runtime path + Astra-specific verification above. Astra / GPT-5.6 / GPT-5.5 / GPT-5.4 all have runtime CRIS alternatives + with their own quotas (`us.`/`global.` prefixed ids; no runtime in-region form) - For `gpt-oss` targets: these do run on `bedrock-runtime`, so standard account TPM limits and the Converse-path mitigations apply - When output-heavy workloads (long JSON, tool outputs, multi-step reasoning) are detected, flag the relevant cap prominently; recommend `bedrock-runtime` for production **only** when the target model actually has a `bedrock-runtime` path diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index cfc5158ee..78fb426c0 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -42,7 +42,9 @@ Two consequences that invert the previous decision logic: 2. **The cost of a same-model move depends on the inference option — state it conditionally.** GPT-5.x In-Region and Geo CRIS are priced at OpenAI's _data residency_ tier, exactly 1.10x the standard list price, so those moves cost ~10% more. **GPT-5.6 on Global CRIS is priced at the standard list price — cost parity** — available only when - the workload has no data-residency constraint (GPT-5.5 / GPT-5.4 have no CRIS at all). See + the workload has no data-residency constraint. GPT-5.5 / GPT-5.4 also have `us.`/`global.` CRIS ids on + `bedrock-runtime`, but their model cards (read 2026-11-06) publish In-Region pricing only, so quote the ~10% + figure for them on every inference option until a Global rate is published. See `shared/openai-on-bedrock.md`. The non-cost case — AWS commitments, IAM/VPC/CloudTrail governance, residency, prompt caching, one vendor relationship — carries the in-region path; never present that path as free or neutral. For Astra, use its separately dated Bedrock rates and verify the source provider's rate before comparing. @@ -72,14 +74,15 @@ GPT-5.4, the target is **the same model on Bedrock**. | GPT-5.6 Sol | GPT-5.6 Sol | `openai.gpt-5.6-sol` / `us.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | | GPT-5.6 Terra | GPT-5.6 Terra | `openai.gpt-5.6-terra` / `us.` `in.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | | GPT-5.6 Luna | GPT-5.6 Luna | `openai.gpt-5.6-luna` / `us.` `in.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | -| GPT-5.5 | GPT-5.5 | `openai.gpt-5.5` (mantle only) | `strong_migrate` — same model, ~10% over OpenAI std | -| GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` (mantle only) | `strong_migrate` — same model, ~10% over OpenAI std | +| GPT-5.5 | GPT-5.5 | `openai.gpt-5.5` / `us.` `global.` prefixed | `strong_migrate` — same model, ~10% over OpenAI std | +| GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` / `us.` `global.` prefixed | `strong_migrate` — same model, ~10% over OpenAI std | Then apply the **region gate** below. Record `model_change: false` and the chosen path in `aws-design-ai.json` → `ai_architecture.code_migration`: `migration_path: "mantle_openai_chat"` for -Astra sources using Chat Completions, or `"mantle_openai_responses"` for a selected Mantle Responses -path. Use `migration_path: "runtime_openai_cris"` when a GPT-6 / GPT-5.6 target is served via -`bedrock-runtime` with a CRIS id (the model cards recommend runtime for new applications). +sources using Chat Completions on a model whose card lists it on mantle (GPT-6, GPT-5.5, GPT-5.4), or +`"mantle_openai_responses"` for a selected Mantle Responses path. Use `migration_path: "runtime_openai_cris"` +when a GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 target is served via `bedrock-runtime` with a CRIS id (the model +cards recommend runtime for new applications). Report the assessment honestly: `strong_migrate` here means "low-risk, well-supported move", not "cheaper". @@ -105,14 +108,16 @@ regions — offer it, stating the data-residency implication of cross-region rou Global routes anywhere and is the cost-parity option). Only a workload that requires strictly in-region processing in a region outside the mantle matrix falls through to Tier 1. -**GPT-5.5 / GPT-5.4:** hard gate. Mantle in-region only, no CRIS, no fallback. If the target region is not in the -matrix: +**GPT-5.5 / GPT-5.4:** same shape as GPT-5.6, with a narrower mantle matrix (GPT-5.5: us-east-1, us-east-2; +GPT-5.4: us-east-1, us-east-2, us-west-2, us-gov-west-1 — per their model cards, read 2026-11-06). If the target +region is in that matrix, either endpoint works. Otherwise the `bedrock-runtime` CRIS path (`us.openai.gpt-5.5` / +`global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`) reaches it, with the same data-residency +caveat as GPT-5.6 (Geo stays within the geography, Global routes anywhere). There is no runtime in-region form. +Only a workload that requires strictly in-region processing in a region outside the mantle matrix falls through: -1. Prefer offering a **region change** to a supported region, if no data-residency constraint pins the workload. - This preserves the zero-behavior-delta benefit, which is the whole point of Tier 0. -2. If the region is fixed, also offer **upgrading within the vendor to GPT-5.6** (which reaches the region via - CRIS) alongside the cross-family option — a generation change, but same vendor and prompt idioms. -3. Otherwise fall through to Tier 1 and say why explicitly — a region constraint, not a model judgment. +1. Prefer offering a **region change** to a supported mantle region, if no data-residency constraint pins the + workload. This preserves the zero-behavior-delta benefit, which is the whole point of Tier 0. +2. Otherwise fall through to Tier 1 and say why explicitly — a region constraint, not a model judgment. Record the outcome in `regional_warnings[]`. @@ -235,10 +240,11 @@ model on price. ### Open-weight gpt-oss -`openai.gpt-oss-120b` (0.15 / 0.60) and `openai.gpt-oss-20b` (0.07 / 0.30) remain available and, unlike the -proprietary GPT models, **do** support `bedrock-runtime` / Converse. They sit a capability class below the GPT-5.x -frontier tier. Recommend them when the user wants OpenAI-architecture models on the Bedrock-native runtime surface -(for Guardrails, invocation logging, or Converse-based tooling) rather than through mantle. +`openai.gpt-oss-120b` (0.15 / 0.60) and `openai.gpt-oss-20b` (0.07 / 0.30) remain available and support +`bedrock-runtime` / Converse with a bare model ID (no CRIS profile needed) and `Invoke`, which the proprietary GPT +models do not. They sit a capability class below the GPT-5.x frontier tier. Recommend them when the user wants +OpenAI-architecture models on the Bedrock-native runtime surface (for Guardrails, invocation logging, or +Converse-based tooling) at the lowest price rather than through mantle. --- @@ -284,7 +290,7 @@ rather than dismissing it. For a long-context (>272K) workload the gap is much w | Chat Completions | Preserve supported Chat when selected | Reshape only for a selected API change; verify the model and endpoint | | Function calling | Supported on GPT-5.x via mantle; Claude tools via Converse | Same-model keeps tool semantics identical | | Reasoning effort | `reasoning={"effort": ...}` on mantle | `none` / `low` / `medium` / `high` / `xhigh` / `max` | -| Prompt caching | GPT-5.6 only (90% off cached input) | Not listed for GPT-5.5 / 5.4; Claude has its own caching | +| Prompt caching | GPT-5.6 (90% off cached input); GPT-5.5 / 5.4 implicit caching on `bedrock-runtime` Responses only | GPT-5.5 / 5.4 cards (read 2026-11-06) list a cache-read rate and implicit caching on runtime, none on mantle; Claude has its own caching | | Streaming | Supported | Verify per surface | | Vision | GPT-5.x (image input) or Claude / Llama 4 | Same-model preserves behavior | | JSON mode | Claude (excellent), Nova Pro (good) | Most models via prompt | @@ -339,11 +345,12 @@ and closed to new customers as of July 30, 2026. ### When to prefer a Bedrock-native model over the mantle GPT path Choose Claude / Nova via Converse when the workload needs Knowledge Bases, intelligent prompt routing, structured -outputs, or application inference profiles — none of which the GPT path offers. Note what no longer forces a family -switch for GPT-5.6: its `bedrock-runtime` CRIS path provides cross-region inference, invocation logs, and -Guardrails (Converse API only; prompt caching is Responses-only on runtime). For GPT-5.5 / GPT-5.4, Guardrails, -logging, and cross-region reach still require a different model. The mantle GPT path is the smallest change; the -Bedrock-native path remains the most feature-complete. +outputs, or application inference profiles — none of which the mantle GPT path offers. Note what no longer forces +a family switch for GPT-5.6, GPT-5.5 and GPT-5.4: their `bedrock-runtime` CRIS paths provide cross-region +inference, invocation logs, Guardrails and application inference profiles (both Converse API only), and +structured outputs and implicit prompt caching (both Responses API only) — per the GPT-5.4 / GPT-5.5 model cards +read 2026-11-06. Knowledge Bases and intelligent prompt routing still require a Bedrock-native model. The mantle +GPT path is the smallest change; the Bedrock-native path remains the most feature-complete. ### High-spend tiering @@ -364,8 +371,8 @@ alternative, not against the source provider. cheaper and exempt from the input-TPM quota, which relieves the binding constraint on mantle. **Very high (>100M tokens/day):** tiering plus caching. Check TPM headroom per model per region early; there is no -RPM quota and no `bedrock-runtime` fallback for GPT models, so quota increases are the scaling lever. See -`shared/ai-migration-guardrails.md`. +RPM quota, and the `bedrock-runtime` CRIS path for GPT-5.6 / 5.5 / 5.4 / GPT-6 carries its own quotas rather than +adding headroom to mantle, so quota increases are the scaling lever. See `shared/ai-migration-guardrails.md`. --- diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md index 18266733c..a4206583c 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md @@ -133,8 +133,8 @@ See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommen | GPT-5.6 Sol | openai.gpt-5.6-sol | OpenAI | 4.40 | 22.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 8.80/33.00) | | GPT-5.6 Terra | openai.gpt-5.6-terra | OpenAI | 2.20 | 13.20 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 4.40/19.80) | | GPT-5.6 Luna | openai.gpt-5.6-luna | OpenAI | 0.22 | 1.32 | 272K | fast | active (mantle in-region + runtime CRIS; 1M tier 0.44/1.98) | -| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle only; no 1M tier) | -| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle only; no 1M tier) | +| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 11.00/49.50; card read 2026-11-06) | +| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 5.50/24.75; card read 2026-11-06) | | Gemma 4 31B | google.gemma-4-31b | Google | 0.14 | 0.40 | 256K | mid | active (Mantle only) | | Gemma 4 26B-A4B | google.gemma-4-26b-a4b | Google | 0.13 | 0.40 | 256K | efficient | active (Mantle only) | | Gemma 4 E2B | google.gemma-4-e2b | Google | 0.04 | 0.08 | 128K | budget | active (Mantle only) | @@ -353,17 +353,24 @@ Regions (mantle): us-east-1 / us-east-2 for all five; us-west-2 additionally car | GPT-5.5 | 5.50 | 33.00 | — | 0.55 | | GPT-5.4 | 2.75 | 16.50 | — | 0.275 | -#### Long context (1M) — GPT-5.6 only, 2.0x input / 1.5x output - -| Model | Input $/1M | Output $/1M | -| ------------- | ---------- | ----------- | -| GPT-5.6 Sol | 8.80 | 33.00 | -| GPT-5.6 Terra | 4.40 | 19.80 | -| GPT-5.6 Luna | 0.44 | 1.98 | - -A workload above 272K context must be priced at the long-context tier. GPT-5.5 and GPT-5.4 have no such tier, so -their usable window is 272K. Global CRIS is published at OpenAI's standard list price (cost parity, GPT-5.6 only — -see `openai-on-bedrock.md`). GovCloud GPT-5.4 is 3.30 / 19.80. +#### Long context (>272K input tokens) — GPT-5.6 at 2.0x input / 1.5x output; GPT-5.5 / GPT-5.4 per their model cards + +| Model | Input $/1M | Output $/1M | Cache read $/1M | +| ------------- | ---------- | ----------- | --------------- | +| GPT-5.6 Sol | 8.80 | 33.00 | — | +| GPT-5.6 Terra | 4.40 | 19.80 | — | +| GPT-5.6 Luna | 0.44 | 1.98 | — | +| GPT-5.5 | 11.00 | 49.50 | 1.10 | +| GPT-5.4 | 5.50 | 24.75 | 0.55 | + +A workload above 272K input tokens must be priced at the long-context tier, applied to all input and output tokens +of the request. GPT-5.5 and GPT-5.4 rows read from their model cards on 2026-11-06 +([GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), +[GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)); both cards state a +1,050,000-token context window (128,000 max output) and list `us.`/`global.` CRIS ids on `bedrock-runtime`, but +publish the In-Region inference option only. Global CRIS is published at OpenAI's standard list price (cost parity, +GPT-5.6 only — see `openai-on-bedrock.md`). GovCloud (US-West) GPT-5.4 short-context is 3.375 / 20.25 (cache read +0.3375) per the 2026-11-06 card; an earlier revision recorded 3.30 / 19.80. > **One source conflict.** These models are absent > from the AWS Price List API entirely: querying `AmazonBedrock` returns only `gpt-oss` and GPT OSS Safeguard, and @@ -373,7 +380,8 @@ see `openai-on-bedrock.md`). GovCloud GPT-5.4 is 3.30 / 19.80. **Prompt caching (GPT-5.6 only):** cached input read at a 90% discount, cache write at 1.25x uncached input, minimum 1,024-token prefix, up to 4 breakpoints, ≥30-minute retention. Cached input is exempt from the input-TPM quota. -GPT-5.5 and GPT-5.4 do not list prompt caching support. +GPT-5.5 and GPT-5.4 list only implicit prompt caching on `bedrock-runtime` Responses (automatic, no request +parameters; cache-read rate above, no cache-write rate) and none on mantle (model cards read 2026-11-06). #### Open-weight gpt-oss diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md index cdf8ced55..7720d4b58 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -190,12 +190,18 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` ## migration_path vocabulary `mantle_openai_responses` (an OpenAI-protocol source — direct or Azure-fronted — with the model -on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `converse` -(Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when a proprietary GPT -model must move off mantle for Guardrails / Knowledge Bases / logging / an unsupported region) · -`gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic -Bedrock SDK swap). A proprietary `openai.gpt-*` model ID is NEVER paired with a -`converse`/`bedrock-runtime` path — those models are mantle-only. +on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `mantle_openai_chat` +(same, preserving a Chat Completions source on a model whose card lists Chat Completions on +mantle — GPT-6, GPT-5.5, GPT-5.4) · `runtime_openai_cris` (same-model move served via +`bedrock-runtime` with a `us.`/`in.`/`global.` CRIS id, `model_change: false`; available for +GPT-6, GPT-5.6, GPT-5.5 and GPT-5.4 per their model cards) · `converse` (Bedrock-native +Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, +intelligent prompt routing, or another feature no same-model path serves) · `gpt-oss` +(OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK +swap). A **bare** proprietary `openai.gpt-*` model ID is NEVER paired with a +`converse`/`bedrock-runtime` path — bare ids are mantle ids; every proprietary GPT target on +`bedrock-runtime` carries a CRIS-prefixed id (there is no runtime in-region form for GPT-6, +GPT-5.6, GPT-5.5 or GPT-5.4). ## regional_validation @@ -255,8 +261,9 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` `price_comparison`. - [ ] `honest_assessment` overall is the weakest across `bedrock_models[]`; if `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. -- [ ] No `bedrock_models[]` entry pairs a proprietary `openai.gpt-*` model ID with a - `converse`/`bedrock-runtime` migration path (mantle-only). +- [ ] No `bedrock_models[]` entry pairs a **bare** proprietary `openai.gpt-*` model ID with a + `converse`/`bedrock-runtime` migration path (bare ids are mantle ids; a `bedrock-runtime` + target for GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 carries a `us.`/`in.`/`global.` CRIS id). - [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves to direct OpenAI: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md b/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md index db0f263c1..18f188d0b 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md @@ -1189,7 +1189,7 @@ Tell the user, before collecting models: > - **OpenAI on Bedrock comes in a few forms, all real** — and which endpoint they use is per-ID, not one blanket rule: > - *Open-weight* `gpt-oss` (`openai.gpt-oss-20b-1:0`, `openai.gpt-oss-120b-1:0`) → `bedrock-runtime` via `bedrock:InvokeModel` / `Converse` (its Responses API is also offered on `bedrock-mantle`). > - *Bare proprietary* GPT ids (`openai.gpt-5*`, not `gpt-oss`) → probed on the `bedrock-mantle` endpoint (Responses API). -> - A *GPT-5.6 CRIS profile* id prefixed `us.` / `in.` / `global.` (e.g. `global.openai.gpt-5.6-sol`) → a `bedrock-runtime` target where Converse is supported. +> - A *CRIS profile* id prefixed `us.` / `in.` / `global.` (e.g. `global.openai.gpt-5.6-sol`, `us.openai.gpt-5.4`, `global.openai.gpt-6-astra`) → a `bedrock-runtime` target where Converse is supported; every proprietary GPT model on Bedrock (GPT-5.6, GPT-6, GPT-5.5, GPT-5.4) has this form, and none has a runtime in-region form. > - The `bedrock-mantle` path uses a **separate** action set — `bedrock-mantle:*` (e.g. the `AmazonBedrockMantleInferenceAccess` managed policy: `bedrock-mantle:CreateInference` + `CallWithBearerToken`) — distinct from `bedrock:InvokeModel`. The preflight decides per model; don't assume. > - What is **not** on Bedrock is calling OpenAI's own hosted API at api.openai.com — that stays with OpenAI. 'GPT on Bedrock' means the AWS-served models above, reached through AWS endpoints and IAM. > @@ -1277,8 +1277,9 @@ Then, per B4's branch table: after they enable it. This is the common, expected outcome for a user who came here to "get access." - `reason: authz` → access is enabled but IAM denies inference. Name the action to grant — for a - standard model `bedrock:InvokeModel`; for a mantle-only `openai.gpt-5*` target (bare proprietary - GPT, not `gpt-oss`) the `bedrock-mantle:*` set (see **B4a**). The `detail` says which — follow it + standard model `bedrock:InvokeModel`; for a bare `openai.gpt-5*` / `openai.gpt-6*` mantle target + (bare proprietary GPT, not `gpt-oss`, not a `us.`/`in.`/`global.` CRIS id) the `bedrock-mantle:*` + set (see **B4a**). The `detail` says which — follow it rather than guessing from the ID, since gpt-oss fails on `bedrock:InvokeModel`, not mantle. - `reason: model_unavailable` → the ID isn't offered in `$REGION`. Use the `resolve-bedrock-model-id` procedure to suggest a cross-region inference-profile ID or a diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/behavior-delta-detection/references/openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/behavior-delta-detection/references/openai-to-bedrock.md index 96ab4f4f7..44c32034d 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/behavior-delta-detection/references/openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/behavior-delta-detection/references/openai-to-bedrock.md @@ -29,7 +29,7 @@ for OpenAI-compatible runtime calls, CRIS ids, and runtime IAM permissions. Astr - `resolution_kind`: `ux_choice` - Source (OpenAI): `client.chat.completions.create(...)`, `messages=[...]`, reads `choices[0].message.content` - Target (mantle GPT): `client.responses.create(...)`, `input=...`, reads `output_text` -- Chat Completions support is **model-specific, not uniformly unverified.** All four GPT-6 models (Astra, Sol, Luna, 6.1 Sol) and the GPT-5.6 family (Sol, Terra, Luna) list `Chat Completions` as a supported API on both `bedrock-mantle` and `bedrock-runtime` CRIS (per their model cards — see `openai-on-bedrock.md` § Access Paths). For these models, a Chat Completions source MAY keep `chat.completions.create` and `choices[0].message` parsing unchanged — **reshape to Responses is OPTIONAL**, driven only by a Responses-only feature (runtime prompt caching, reasoning-item round-trips on `bedrock-runtime`) or a demonstrated account-specific constraint, not by a blanket assumption that Chat Completions is unsupported. For GPT-5.5 / GPT-5.4 (mantle-only, no CRIS) Chat Completions support is still unconfirmed on their model cards — treat those two as requiring a reshape to Responses, and probe the target account before committing. +- Chat Completions support is **model-specific, not uniformly unverified.** All four GPT-6 models (Astra, Sol, Luna, 6.1 Sol) and the GPT-5.6 family (Sol, Terra, Luna) list `Chat Completions` as a supported API on both `bedrock-mantle` and `bedrock-runtime` CRIS (per their model cards — see `openai-on-bedrock.md` § Access Paths). For these models, a Chat Completions source MAY keep `chat.completions.create` and `choices[0].message` parsing unchanged — **reshape to Responses is OPTIONAL**, driven only by a Responses-only feature (runtime prompt caching, reasoning-item round-trips on `bedrock-runtime`) or a demonstrated account-specific constraint, not by a blanket assumption that Chat Completions is unsupported. GPT-5.5 / GPT-5.4 follow the same rule: their model cards (read 2026-11-06 — [GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), [GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)) list `Chat Completions` as supported on both `bedrock-mantle` and `bedrock-runtime` (`us.`/`global.` CRIS ids), so the reshape is OPTIONAL for them too. Plan a reshape only when the selected model's card leaves Chat Completions unchecked on the selected endpoint, and probe the target account before committing, as for any model. ### detect_grep @@ -66,7 +66,7 @@ grep -rEn 'function_call_output|tool_call' --include="*.py" --include="*. ## prompt-caching-availability - `resolution_kind`: `impl_path` -- Prompt caching is listed as supported on GPT-5.6 and GPT-6 only. Do not emit caching configuration for `openai.gpt-5.5` or `openai.gpt-5.4`. +- Prompt caching with request parameters (explicit mode) is listed on GPT-5.6 and GPT-6 only. The GPT-5.5 / GPT-5.4 cards (read 2026-11-06) list only implicit caching on `bedrock-runtime` Responses (automatic, no request parameters) and none on mantle — do not emit caching configuration for `openai.gpt-5.5` or `openai.gpt-5.4` on either endpoint. - Astra lists implicit and explicit prompt caching on mantle Responses, with **30-minute cache-write** rates. Do not inherit GPT-5.6's minimum-prefix, breakpoint, or cached-input quota-exemption rules. diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/resolve-bedrock-model-id/resolve-bedrock-model-id.md b/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/resolve-bedrock-model-id/resolve-bedrock-model-id.md index f64dc9fef..f2da1fd11 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/resolve-bedrock-model-id/resolve-bedrock-model-id.md +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/resolve-bedrock-model-id/resolve-bedrock-model-id.md @@ -50,9 +50,13 @@ aws bedrock list-foundation-models \ **Check this before Step 1.** Route by family and endpoint (GPT-5 verified 2026-08-21; Astra regions verified 2026-09-30; see `gcp-to-aws/references/shared/openai-on-bedrock.md`): -**Case A — GPT-5.5 / GPT-5.4 (`openai.gpt-5.5`, `openai.gpt-5.4`): mantle-only, no inference profile.** The -inference-profile path below cannot resolve them — `list-inference-profiles` never returns them, and Step 3's token -ranking would end in a spurious `blocked` for a perfectly valid id. Validate against the model catalog instead: +**Case A — bare GPT-5.5 / GPT-5.4 ids (`openai.gpt-5.5`, `openai.gpt-5.4`) when the plan targets the mantle +endpoint: bare mantle id, not an inference profile.** The inference-profile path below cannot resolve the bare form — +`list-inference-profiles` returns only the CRIS-prefixed ids (`us.openai.gpt-5.5`, `global.openai.gpt-5.5`, +`us.openai.gpt-5.4`, `global.openai.gpt-5.4`, per the model cards read 2026-11-06), never the bare mantle id — and +Step 3's token ranking would end in a spurious `blocked` for a perfectly valid id. (When the plan targets +`bedrock-runtime` for GPT-5.5 / GPT-5.4, the CRIS id flows through Case B and Steps 1–5 like GPT-5.6.) Validate the +bare id against the model catalog instead: ```bash aws bedrock list-foundation-models \ @@ -69,7 +73,7 @@ aws bedrock list-foundation-models \ this catalog's `modelId` + `modelName` pairs instead of against inference profiles. **Do this check before falling through to Step 1** — a friendly name like `"OpenAI GPT-5.5"` (tokens `{openai, gpt, 5}`) has full token overlap with `openai.gpt-5.5`'s own id (`{openai, gpt, 5}`) and would otherwise fall through to Step 1's - `list-inference-profiles`, which never lists a mantle-only model — the free-text input would then reach Step 3's + `list-inference-profiles`, which never lists a bare mantle id — the free-text input would then reach Step 3's ranking with zero candidates and dead-end at `blocked` with no matches shown, even though the foundation catalog had the exact model the whole time. A single candidate with full token overlap on the model-identifying tokens (excluding generic tokens like a bare `gpt`) → surface it as the ONE candidate through Step 4 exactly as an @@ -78,24 +82,27 @@ aws bedrock list-foundation-models \ list alongside any inference-profile candidates Steps 1–3 separately produce; do not silently drop either source. - **No match at all (exact or token) and `plan_model_id` looked ID-shaped, not free-text** → not enabled or not available in this region. Return `blocked` with `reason: model_unresolvable`, putting the region and the - `openai.*` ids that _were_ returned in `detail`. These two models have no CRIS, so the remedy is a region change - or a different model — never an inference-profile prefix. + `openai.*` ids that _were_ returned in `detail`. Do not silently add an inference-profile prefix to rescue a + mantle plan: the remedy is a region change, a different model, or a plan change to the `bedrock-runtime` CRIS + path (`us.` / `global.` ids, Case B) — a decision for the plan, not for this resolver. - CLI failure / missing `bedrock:ListFoundationModels` → `blocked` with `reason: model_unresolvable` and the exact error in `detail`, rather than guessing. -These two need `bedrock-mantle:*` IAM actions (e.g. `AmazonBedrockMantleInferenceAccess`), not +The bare ids need `bedrock-mantle:*` IAM actions (e.g. `AmazonBedrockMantleInferenceAccess`), not `bedrock:InvokeModel`; resolution success does not imply invoke authorization. **A free-text name that does NOT token-match this catalog** (e.g. `"anthropic claude haiku"`, `"Nova Micro"`) -continues to Step 1 as before — this catalog check only short-circuits the specific case a mantle-only model would +continues to Step 1 as before — this catalog check only short-circuits the specific case a bare mantle id would otherwise be unreachable from free text; it is not a replacement for Step 1's inference-profile search for every other provider/model. -**Case B — GPT-5.6 and GPT-6 (`openai.gpt-5.6-sol` / `-terra` / `-luna`, `openai.gpt-6-astra` / `-sol` / `-luna`, -`openai.gpt-6.1-sol`, or already `us.` / `in.` / `global.` prefixed): BOTH paths exist.** The bare id is the mantle -form; `bedrock-runtime` serves these models through CRIS inference profiles (`us.openai.gpt-5.6-*` / -`us.openai.gpt-6-*`, `in.openai.gpt-5.6-*` in India Regions, `global.openai.gpt-5.6-*` / `global.openai.gpt-6-*`), -which `list-inference-profiles` DOES return. Route on the plan's intent: +**Case B — GPT-5.6, GPT-6, and GPT-5.5 / GPT-5.4 on `bedrock-runtime` (`openai.gpt-5.6-sol` / `-terra` / `-luna`, +`openai.gpt-6-astra` / `-sol` / `-luna`, `openai.gpt-6.1-sol`, `openai.gpt-5.5` / `openai.gpt-5.4` with a runtime +plan, or already `us.` / `in.` / `global.` prefixed): BOTH paths exist.** The bare id is the mantle form; +`bedrock-runtime` serves these models through CRIS inference profiles (`us.openai.gpt-5.6-*` / `us.openai.gpt-6-*` / +`us.openai.gpt-5.5` / `us.openai.gpt-5.4`, `in.openai.gpt-5.6-*` in India Regions, `global.openai.gpt-5.6-*` / +`global.openai.gpt-6-*` / `global.openai.gpt-5.5` / `global.openai.gpt-5.4`), which `list-inference-profiles` DOES +return. Route on the plan's intent: - Plan targets the mantle endpoint (`migration_path` starts with `mantle`, or the id is bare) → validate the bare id against the model catalog exactly as in Case A. @@ -217,9 +224,9 @@ stops on abort. - This skill is idempotent: calling it twice with the same already-validated ID will hit Step 0 (documented bare-ID route) or Step 2 (inference profile) and return immediately. - Steps 1–5 assume the target is a `bedrock-runtime` model reachable through an - inference profile. Mantle-only ids (GPT-5.5/5.4, and GPT-5.6 when the plan - targets the mantle endpoint), and London in-region Opus 5.5, are handled entirely in Step 0; GPT-5.6/GPT-6 CRIS ids - flow through Steps 1–5 like any other inference profile. See + inference profile. Bare mantle ids (any proprietary GPT model — GPT-5.5/5.4, GPT-5.6, GPT-6 — when the plan + targets the mantle endpoint), and London in-region Opus 5.5, are handled entirely in Step 0; GPT-5.6/GPT-6 and + GPT-5.5/5.4 CRIS ids flow through Steps 1–5 like any other inference profile. See `$ASSESS_BASE/references/shared/openai-on-bedrock.md` for the authoritative family split. - Output of this skill should replace the plan's `target_model_id` in the diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/bedrock_pricing.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/bedrock_pricing.py index 7ab37804b..6530c758b 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/bedrock_pricing.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/bedrock_pricing.py @@ -90,7 +90,11 @@ def parse_price_dimensions(price_item: dict) -> dict: # price — cost PARITY, for workloads with no residency constraint # The GPT-5.6 family also has a LONG-CONTEXT (1M) tier at 2.0x input / 1.5x # output per option, NOT represented here — a >272K workload priced from this - # table is understated. GPT-5.5 and GPT-5.4: mantle-only, no CRIS, no 1M tier. + # table is understated. GPT-5.5 and GPT-5.4 (model cards read 2026-11-06) also + # have a long-context (>272K) tier (5.5: 11.00/49.50; 5.4: 5.50/24.75 per 1M), + # likewise NOT represented here. Their cards publish the In-Region option only; + # the us./global. CRIS ids they list on bedrock-runtime carry no separate price + # row, so no prefixed GPT-5.5/5.4 entries are added below until one is published. "openai.gpt-5.6-sol": {"input_per_1k_usd": 0.0044, "output_per_1k_usd": 0.022}, "openai.gpt-5.6-terra": {"input_per_1k_usd": 0.0022, "output_per_1k_usd": 0.0132}, "openai.gpt-5.6-luna": {"input_per_1k_usd": 0.00022, "output_per_1k_usd": 0.00132}, @@ -135,9 +139,12 @@ def parse_price_dimensions(price_item: dict) -> dict: def is_mantle_gpt(model_id: str) -> bool: - """Pure: OpenAI's proprietary GPT models (GPT-5.5/5.4/5.6 and GPT-6 - Astra/Sol/Luna/6.1 Sol), which the AWS PriceList API does not carry. The - open-weight gpt-oss models ARE in the PriceList API and must not match.""" + """Pure: BARE ids of OpenAI's proprietary GPT models (GPT-5.5/5.4/5.6 and GPT-6 + Astra/Sol/Luna/6.1 Sol) — the bedrock-mantle id form — which the AWS PriceList + API does not carry. The `us.`/`in.`/`global.` CRIS forms (bedrock-runtime; GPT-5.5 + and 5.4 have `us.`/`global.` ids too) do not match here; callers pair this with + an explicit prefixed-id regex. The open-weight gpt-oss models ARE in the + PriceList API and must not match.""" mid = model_id.lower() return (mid.startswith("openai.gpt-5") or mid.startswith("openai.gpt-6")) and "oss" not in mid diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py index 5b56d2fd2..0ceaa4f1e 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py @@ -20,12 +20,14 @@ def is_inference_profile(model_id: str) -> bool: def is_mantle_model(model_id: str, region: str = "") -> bool: """True for bare proprietary GPT-5/GPT-6 IDs and the bare Opus 5.5 ID, which use - bedrock-mantle (Responses and Messages respectively). They need + bedrock-mantle (Responses/Chat Completions and Messages respectively). They need `bedrock-mantle:*` actions — a policy granting only `bedrock:InvokeModel` against a foundation-model ARN cannot authorize these bare IDs. Opus 5.5 in eu-west-2 is a runtime in-region target (model card verified 2026-10-03); its other bare-ID regions use Mantle. - Their prefixed forms use runtime instead. + The `us.`/`in.`/`global.` prefixed forms use bedrock-runtime instead — for every + proprietary GPT model, GPT-5.5/5.4 included (their model cards list `us.`/`global.` + CRIS ids; none has a runtime in-region form), so a prefixed id must NOT match. The open-weight gpt-oss models DO use bedrock-runtime and must not match.""" mid = model_id.lower() return ((mid.startswith("openai.gpt-5") or mid.startswith("openai.gpt-6")) and "oss" not in mid) or ( diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py index 2a4c031bb..fde343ece 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py @@ -60,12 +60,15 @@ def is_embedding_model(model_id: str) -> bool: def is_mantle_model(model_id: str) -> bool: """Pure: ids that must be probed via the mantle Responses API rather than Converse/InvokeModel. Matches the BARE proprietary GPT ids (`openai.gpt-5*`, - `openai.gpt-6*` — covers GPT-5.5/5.4/5.6 and GPT-6 Astra/Sol/Luna/6.1 Sol), - which are mantle-served. Deliberately does NOT match: + `openai.gpt-6*` — covers GPT-5.5/5.4/5.6 and GPT-6 Astra/Sol/Luna/6.1 Sol). + The bare id is the bedrock-mantle form for every proprietary GPT model; none + of them has a bedrock-runtime in-region form. Deliberately does NOT match: - gpt-oss ids (Converse-capable on bedrock-runtime), and - - GPT-5.6/GPT-6 CRIS profile ids (`us.`/`in.`/`global.` prefixed) — those are - bedrock-runtime targets where Converse IS supported, so falling through to - the standard Converse probe is the correct behavior, not an accident.""" + - CRIS profile ids (`us.`/`in.`/`global.` prefixed) for any proprietary GPT + model, GPT-5.5/5.4 included (their model cards list `us.`/`global.` ids on + bedrock-runtime) — those are bedrock-runtime targets where Converse IS + supported, so falling through to the standard Converse probe is the + correct behavior, not an accident.""" mid = model_id.lower() return (mid.startswith("openai.gpt-5") or mid.startswith("openai.gpt-6")) and "oss" not in mid @@ -160,8 +163,8 @@ def classify_mantle_error(status: int | None, message: str) -> dict: f"Mantle is in-region only; for GPT-5.6 try a bedrock-runtime CRIS " f"id (us./in./global. prefixed); for GPT-6 Astra use us-east-1/us-west-2 " f"mantle or a supported us./global. runtime CRIS path. " - f"For GPT-5.5/5.4 switch to " - f"a supported region or a different model."} + f"For GPT-5.5/5.4 switch to a supported mantle region, a " + f"us./global. bedrock-runtime CRIS id, or a different model."} if status == 429: # Reaching a token-per-minute ceiling still proves we are authorized. return {"ok": True, "reason": "throttled_ok", diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_bedrock_pricing.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_bedrock_pricing.py index 40be0ef0e..55161d2a7 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_bedrock_pricing.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_bedrock_pricing.py @@ -295,6 +295,18 @@ def test_mantle_gpt_detection_excludes_gpt_oss(): assert bp.is_mantle_gpt("anthropic.claude-sonnet-4-6") is False +def test_mantle_gpt_detection_gpt54_55_bare_is_mantle_cris_is_runtime(): + # GPT-5.4 / GPT-5.5 model cards (read 2026-11-06) list us./global. CRIS ids on + # bedrock-runtime. is_mantle_gpt matches only the bare (mantle) id form; the + # prefixed runtime forms are handled by the callers' explicit CRIS regex. + assert bp.is_mantle_gpt("openai.gpt-5.4") is True + assert bp.is_mantle_gpt("openai.gpt-5.5") is True + assert bp.is_mantle_gpt("us.openai.gpt-5.4") is False + assert bp.is_mantle_gpt("global.openai.gpt-5.4") is False + assert bp.is_mantle_gpt("us.openai.gpt-5.5") is False + assert bp.is_mantle_gpt("global.openai.gpt-5.5") is False + + def test_mantle_gpt_detection_includes_bare_gpt6(): # Finding 1: bare GPT-6 ids must be recognized by the same detector that # routes GPT-5.x to the curated static table / mantle path. diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py index 7544ad18c..b3f6449ab 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py @@ -126,6 +126,18 @@ def test_mantle_model_detection_excludes_gpt_oss(): assert is_mantle_model("anthropic.claude-sonnet-4-6") is False +def test_is_mantle_model_gpt54_55_cris_ids_are_runtime_not_mantle(): + # GPT-5.4 / GPT-5.5 model cards (read 2026-11-06) list us./global. CRIS ids on + # bedrock-runtime: the bare id needs bedrock-mantle:* actions, the prefixed id + # needs bedrock:InvokeModel against the inference-profile ARN. + assert is_mantle_model("openai.gpt-5.4") is True + assert is_mantle_model("openai.gpt-5.5") is True + assert is_mantle_model("us.openai.gpt-5.4") is False + assert is_mantle_model("global.openai.gpt-5.4") is False + assert is_mantle_model("us.openai.gpt-5.5") is False + assert is_mantle_model("global.openai.gpt-5.5") is False + + def test_is_mantle_model_matches_bare_gpt6_ids(): assert is_mantle_model("openai.gpt-6-astra") is True assert is_mantle_model("openai.gpt-6-sol") is True diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py index 71f526976..3056e8b0a 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py @@ -197,6 +197,18 @@ def test_mantle_model_detection_excludes_gpt_oss_and_cris_forms(): assert p.is_mantle_model("amazon.nova-lite-v1:0") is False +def test_is_mantle_model_gpt54_55_cris_ids_are_runtime_not_mantle(): + # GPT-5.4 / GPT-5.5 model cards (read 2026-11-06) list us./global. CRIS ids on + # bedrock-runtime. The ID-form rule already classifies them: bare id -> mantle + # probe, CRIS-prefixed id -> Converse probe. Lock that in for these two models. + assert p.is_mantle_model("openai.gpt-5.4") is True + assert p.is_mantle_model("openai.gpt-5.5") is True + assert p.is_mantle_model("us.openai.gpt-5.4") is False + assert p.is_mantle_model("global.openai.gpt-5.4") is False + assert p.is_mantle_model("us.openai.gpt-5.5") is False + assert p.is_mantle_model("global.openai.gpt-5.5") is False + + def test_is_mantle_model_matches_bare_gpt6_ids(): # Bare GPT-6 ids (Astra, Sol, Luna, 6.1 Sol) are mantle-served, same as the # bare GPT-5.x ids above. Their CRIS forms (us./global. prefixed) stay on the @@ -221,17 +233,20 @@ def test_mantle_authz_error_points_at_mantle_actions_not_invoke_model(): def test_mantle_model_unavailable_remedy_is_family_split(): - # Verified 2026-08-21: mantle itself is in-region only, but GPT-5.6 now has a - # bedrock-runtime CRIS path — so the 404 remedy must offer the CRIS form for - # 5.6 while making clear 5.5/5.4 have no prefixed form. An earlier version of - # this test asserted the opposite (never suggest a prefix), which matched the - # pre-2026-08-21 docs. + # Mantle itself is in-region only, but every proprietary GPT model has a + # bedrock-runtime CRIS path — GPT-5.6 (us./in./global., verified 2026-08-21) + # and GPT-5.5/5.4 (us./global., model cards read 2026-11-06). The 404 remedy + # must offer the CRIS form for all of them. An earlier version of this test + # asserted that 5.5/5.4 have no prefixed form, which matched the pre-2026-11-06 + # model cards. v = p.classify_mantle_error(404, "model not found") assert v["ok"] is False assert v["reason"] == "model_unavailable" assert "in-region only" in v["detail"] assert "CRIS" in v["detail"] and "us./in./global." in v["detail"] assert "GPT-5.5/5.4" in v["detail"] + assert "us./global. bedrock-runtime CRIS id" in v["detail"] + assert "different model" in v["detail"] def test_mantle_throttle_is_ok_for_preflight(): v = p.classify_mantle_error(429, "too many tokens per minute") diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-migration-guardrails.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-migration-guardrails.md index a819dc895..1ad1b0a09 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-migration-guardrails.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-migration-guardrails.md @@ -48,9 +48,9 @@ Inference on `bedrock-mantle` for these models is governed by **two per-model, p | Medium | Medium | Monitor 429s against **token** throughput, not request rate; enable prompt caching | | High | High | Enable prompt caching first (cached input is exempt from input TPM), then request a quota increase | -**The `bedrock-runtime` fallback exists only for GPT-5.6 and GPT-6 (Astra, Sol, Luna, 6.1 Sol, 6.1 Terra).** GPT-5.5 and GPT-5.4 are `bedrock-mantle` only and in-region only — for them, "switch to `bedrock-runtime`" requires moving to a different model (Bedrock-native or `gpt-oss`), a model change with its own eval cost. GPT-5.6 Sol/Terra/Luna and GPT-6 Astra/Sol/Luna/6.1 Sol/6.1 Terra DO have a `bedrock-runtime` path via CRIS inference profiles (GPT-5.6 `us.`/`in.`/`global.`, GPT-6 `us.`/`global.` prefixed ids; the model cards recommend runtime for new applications) — a legitimate endpoint option with its own quota family, and on Global CRIS it is also the cost-parity option. Scaling levers on the mantle path, in order: +**Every proprietary GPT model on Bedrock has a `bedrock-runtime` path via CRIS inference profiles.** GPT-5.6 Sol/Terra/Luna (`us.`/`in.`/`global.`), GPT-6 Astra/Sol/Luna/6.1 Sol/6.1 Terra (`us.`/`global.`), and GPT-5.5 / GPT-5.4 (`us.`/`global.`; per their model cards read 2026-11-06, [GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), [GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)) all list `bedrock-runtime` with Responses, Chat Completions and Converse; none has a runtime in-region form. The model cards recommend runtime for new applications. It is a legitimate endpoint option with its own quota family, and for GPT-5.6 / GPT-6 on Global CRIS it is also the cost-parity option (the GPT-5.5 / GPT-5.4 cards publish In-Region pricing only). A same-model endpoint switch is not a model change; only moving to a Bedrock-native model or `gpt-oss` carries the eval cost of a model change. Scaling levers on the mantle path, in order: -1. Prompt caching (GPT-5.6 and GPT-6 only) — 90% off cached input and exempt from the input-TPM quota +1. Prompt caching (GPT-5.6 and GPT-6 on mantle; GPT-5.5 / GPT-5.4 list implicit caching on `bedrock-runtime` Responses only) — 90% off cached input and exempt from the input-TPM quota 2. Exponential backoff with a bounded retry count (`max_retries` on the OpenAI SDK) 3. Spreading load across minutes and ramping request rate gradually rather than bursting 4. A Service Quotas increase for that model's input/output TPM in that region @@ -108,8 +108,8 @@ Claude models on Mantle have an additional **output TPM cap** that differs by mo - For Claude migrations at medium/high volume: the 2M output TPM cap on Claude 4.7+ is the binding constraint - For OpenAI proprietary GPT targets: this Claude cap does not apply. Use the GPT-5.x mantle quotas or - Astra-specific verification above. Astra / GPT-5.6 have runtime CRIS alternatives with their own - quotas; GPT-5.5/5.4 have no runtime path + Astra-specific verification above. Astra / GPT-5.6 / GPT-5.5 / GPT-5.4 all have runtime CRIS alternatives + with their own quotas (`us.`/`global.` prefixed ids; no runtime in-region form) - For `gpt-oss` targets: these do run on `bedrock-runtime`, so standard account TPM limits and the Converse-path mitigations apply - When output-heavy workloads (long JSON, tool outputs, multi-step reasoning) are detected, flag the relevant cap prominently; recommend `bedrock-runtime` for production **only** when the target model actually has a `bedrock-runtime` path diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md index cfc5158ee..78fb426c0 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md @@ -42,7 +42,9 @@ Two consequences that invert the previous decision logic: 2. **The cost of a same-model move depends on the inference option — state it conditionally.** GPT-5.x In-Region and Geo CRIS are priced at OpenAI's _data residency_ tier, exactly 1.10x the standard list price, so those moves cost ~10% more. **GPT-5.6 on Global CRIS is priced at the standard list price — cost parity** — available only when - the workload has no data-residency constraint (GPT-5.5 / GPT-5.4 have no CRIS at all). See + the workload has no data-residency constraint. GPT-5.5 / GPT-5.4 also have `us.`/`global.` CRIS ids on + `bedrock-runtime`, but their model cards (read 2026-11-06) publish In-Region pricing only, so quote the ~10% + figure for them on every inference option until a Global rate is published. See `shared/openai-on-bedrock.md`. The non-cost case — AWS commitments, IAM/VPC/CloudTrail governance, residency, prompt caching, one vendor relationship — carries the in-region path; never present that path as free or neutral. For Astra, use its separately dated Bedrock rates and verify the source provider's rate before comparing. @@ -72,14 +74,15 @@ GPT-5.4, the target is **the same model on Bedrock**. | GPT-5.6 Sol | GPT-5.6 Sol | `openai.gpt-5.6-sol` / `us.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | | GPT-5.6 Terra | GPT-5.6 Terra | `openai.gpt-5.6-terra` / `us.` `in.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | | GPT-5.6 Luna | GPT-5.6 Luna | `openai.gpt-5.6-luna` / `us.` `in.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | -| GPT-5.5 | GPT-5.5 | `openai.gpt-5.5` (mantle only) | `strong_migrate` — same model, ~10% over OpenAI std | -| GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` (mantle only) | `strong_migrate` — same model, ~10% over OpenAI std | +| GPT-5.5 | GPT-5.5 | `openai.gpt-5.5` / `us.` `global.` prefixed | `strong_migrate` — same model, ~10% over OpenAI std | +| GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` / `us.` `global.` prefixed | `strong_migrate` — same model, ~10% over OpenAI std | Then apply the **region gate** below. Record `model_change: false` and the chosen path in `aws-design-ai.json` → `ai_architecture.code_migration`: `migration_path: "mantle_openai_chat"` for -Astra sources using Chat Completions, or `"mantle_openai_responses"` for a selected Mantle Responses -path. Use `migration_path: "runtime_openai_cris"` when a GPT-6 / GPT-5.6 target is served via -`bedrock-runtime` with a CRIS id (the model cards recommend runtime for new applications). +sources using Chat Completions on a model whose card lists it on mantle (GPT-6, GPT-5.5, GPT-5.4), or +`"mantle_openai_responses"` for a selected Mantle Responses path. Use `migration_path: "runtime_openai_cris"` +when a GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 target is served via `bedrock-runtime` with a CRIS id (the model +cards recommend runtime for new applications). Report the assessment honestly: `strong_migrate` here means "low-risk, well-supported move", not "cheaper". @@ -105,14 +108,16 @@ regions — offer it, stating the data-residency implication of cross-region rou Global routes anywhere and is the cost-parity option). Only a workload that requires strictly in-region processing in a region outside the mantle matrix falls through to Tier 1. -**GPT-5.5 / GPT-5.4:** hard gate. Mantle in-region only, no CRIS, no fallback. If the target region is not in the -matrix: +**GPT-5.5 / GPT-5.4:** same shape as GPT-5.6, with a narrower mantle matrix (GPT-5.5: us-east-1, us-east-2; +GPT-5.4: us-east-1, us-east-2, us-west-2, us-gov-west-1 — per their model cards, read 2026-11-06). If the target +region is in that matrix, either endpoint works. Otherwise the `bedrock-runtime` CRIS path (`us.openai.gpt-5.5` / +`global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`) reaches it, with the same data-residency +caveat as GPT-5.6 (Geo stays within the geography, Global routes anywhere). There is no runtime in-region form. +Only a workload that requires strictly in-region processing in a region outside the mantle matrix falls through: -1. Prefer offering a **region change** to a supported region, if no data-residency constraint pins the workload. - This preserves the zero-behavior-delta benefit, which is the whole point of Tier 0. -2. If the region is fixed, also offer **upgrading within the vendor to GPT-5.6** (which reaches the region via - CRIS) alongside the cross-family option — a generation change, but same vendor and prompt idioms. -3. Otherwise fall through to Tier 1 and say why explicitly — a region constraint, not a model judgment. +1. Prefer offering a **region change** to a supported mantle region, if no data-residency constraint pins the + workload. This preserves the zero-behavior-delta benefit, which is the whole point of Tier 0. +2. Otherwise fall through to Tier 1 and say why explicitly — a region constraint, not a model judgment. Record the outcome in `regional_warnings[]`. @@ -235,10 +240,11 @@ model on price. ### Open-weight gpt-oss -`openai.gpt-oss-120b` (0.15 / 0.60) and `openai.gpt-oss-20b` (0.07 / 0.30) remain available and, unlike the -proprietary GPT models, **do** support `bedrock-runtime` / Converse. They sit a capability class below the GPT-5.x -frontier tier. Recommend them when the user wants OpenAI-architecture models on the Bedrock-native runtime surface -(for Guardrails, invocation logging, or Converse-based tooling) rather than through mantle. +`openai.gpt-oss-120b` (0.15 / 0.60) and `openai.gpt-oss-20b` (0.07 / 0.30) remain available and support +`bedrock-runtime` / Converse with a bare model ID (no CRIS profile needed) and `Invoke`, which the proprietary GPT +models do not. They sit a capability class below the GPT-5.x frontier tier. Recommend them when the user wants +OpenAI-architecture models on the Bedrock-native runtime surface (for Guardrails, invocation logging, or +Converse-based tooling) at the lowest price rather than through mantle. --- @@ -284,7 +290,7 @@ rather than dismissing it. For a long-context (>272K) workload the gap is much w | Chat Completions | Preserve supported Chat when selected | Reshape only for a selected API change; verify the model and endpoint | | Function calling | Supported on GPT-5.x via mantle; Claude tools via Converse | Same-model keeps tool semantics identical | | Reasoning effort | `reasoning={"effort": ...}` on mantle | `none` / `low` / `medium` / `high` / `xhigh` / `max` | -| Prompt caching | GPT-5.6 only (90% off cached input) | Not listed for GPT-5.5 / 5.4; Claude has its own caching | +| Prompt caching | GPT-5.6 (90% off cached input); GPT-5.5 / 5.4 implicit caching on `bedrock-runtime` Responses only | GPT-5.5 / 5.4 cards (read 2026-11-06) list a cache-read rate and implicit caching on runtime, none on mantle; Claude has its own caching | | Streaming | Supported | Verify per surface | | Vision | GPT-5.x (image input) or Claude / Llama 4 | Same-model preserves behavior | | JSON mode | Claude (excellent), Nova Pro (good) | Most models via prompt | @@ -339,11 +345,12 @@ and closed to new customers as of July 30, 2026. ### When to prefer a Bedrock-native model over the mantle GPT path Choose Claude / Nova via Converse when the workload needs Knowledge Bases, intelligent prompt routing, structured -outputs, or application inference profiles — none of which the GPT path offers. Note what no longer forces a family -switch for GPT-5.6: its `bedrock-runtime` CRIS path provides cross-region inference, invocation logs, and -Guardrails (Converse API only; prompt caching is Responses-only on runtime). For GPT-5.5 / GPT-5.4, Guardrails, -logging, and cross-region reach still require a different model. The mantle GPT path is the smallest change; the -Bedrock-native path remains the most feature-complete. +outputs, or application inference profiles — none of which the mantle GPT path offers. Note what no longer forces +a family switch for GPT-5.6, GPT-5.5 and GPT-5.4: their `bedrock-runtime` CRIS paths provide cross-region +inference, invocation logs, Guardrails and application inference profiles (both Converse API only), and +structured outputs and implicit prompt caching (both Responses API only) — per the GPT-5.4 / GPT-5.5 model cards +read 2026-11-06. Knowledge Bases and intelligent prompt routing still require a Bedrock-native model. The mantle +GPT path is the smallest change; the Bedrock-native path remains the most feature-complete. ### High-spend tiering @@ -364,8 +371,8 @@ alternative, not against the source provider. cheaper and exempt from the input-TPM quota, which relieves the binding constraint on mantle. **Very high (>100M tokens/day):** tiering plus caching. Check TPM headroom per model per region early; there is no -RPM quota and no `bedrock-runtime` fallback for GPT models, so quota increases are the scaling lever. See -`shared/ai-migration-guardrails.md`. +RPM quota, and the `bedrock-runtime` CRIS path for GPT-5.6 / 5.5 / 5.4 / GPT-6 carries its own quotas rather than +adding headroom to mantle, so quota increases are the scaling lever. See `shared/ai-migration-guardrails.md`. --- diff --git a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md index 18266733c..a4206583c 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md @@ -133,8 +133,8 @@ See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommen | GPT-5.6 Sol | openai.gpt-5.6-sol | OpenAI | 4.40 | 22.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 8.80/33.00) | | GPT-5.6 Terra | openai.gpt-5.6-terra | OpenAI | 2.20 | 13.20 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 4.40/19.80) | | GPT-5.6 Luna | openai.gpt-5.6-luna | OpenAI | 0.22 | 1.32 | 272K | fast | active (mantle in-region + runtime CRIS; 1M tier 0.44/1.98) | -| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle only; no 1M tier) | -| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle only; no 1M tier) | +| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 11.00/49.50; card read 2026-11-06) | +| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 5.50/24.75; card read 2026-11-06) | | Gemma 4 31B | google.gemma-4-31b | Google | 0.14 | 0.40 | 256K | mid | active (Mantle only) | | Gemma 4 26B-A4B | google.gemma-4-26b-a4b | Google | 0.13 | 0.40 | 256K | efficient | active (Mantle only) | | Gemma 4 E2B | google.gemma-4-e2b | Google | 0.04 | 0.08 | 128K | budget | active (Mantle only) | @@ -353,17 +353,24 @@ Regions (mantle): us-east-1 / us-east-2 for all five; us-west-2 additionally car | GPT-5.5 | 5.50 | 33.00 | — | 0.55 | | GPT-5.4 | 2.75 | 16.50 | — | 0.275 | -#### Long context (1M) — GPT-5.6 only, 2.0x input / 1.5x output - -| Model | Input $/1M | Output $/1M | -| ------------- | ---------- | ----------- | -| GPT-5.6 Sol | 8.80 | 33.00 | -| GPT-5.6 Terra | 4.40 | 19.80 | -| GPT-5.6 Luna | 0.44 | 1.98 | - -A workload above 272K context must be priced at the long-context tier. GPT-5.5 and GPT-5.4 have no such tier, so -their usable window is 272K. Global CRIS is published at OpenAI's standard list price (cost parity, GPT-5.6 only — -see `openai-on-bedrock.md`). GovCloud GPT-5.4 is 3.30 / 19.80. +#### Long context (>272K input tokens) — GPT-5.6 at 2.0x input / 1.5x output; GPT-5.5 / GPT-5.4 per their model cards + +| Model | Input $/1M | Output $/1M | Cache read $/1M | +| ------------- | ---------- | ----------- | --------------- | +| GPT-5.6 Sol | 8.80 | 33.00 | — | +| GPT-5.6 Terra | 4.40 | 19.80 | — | +| GPT-5.6 Luna | 0.44 | 1.98 | — | +| GPT-5.5 | 11.00 | 49.50 | 1.10 | +| GPT-5.4 | 5.50 | 24.75 | 0.55 | + +A workload above 272K input tokens must be priced at the long-context tier, applied to all input and output tokens +of the request. GPT-5.5 and GPT-5.4 rows read from their model cards on 2026-11-06 +([GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), +[GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)); both cards state a +1,050,000-token context window (128,000 max output) and list `us.`/`global.` CRIS ids on `bedrock-runtime`, but +publish the In-Region inference option only. Global CRIS is published at OpenAI's standard list price (cost parity, +GPT-5.6 only — see `openai-on-bedrock.md`). GovCloud (US-West) GPT-5.4 short-context is 3.375 / 20.25 (cache read +0.3375) per the 2026-11-06 card; an earlier revision recorded 3.30 / 19.80. > **One source conflict.** These models are absent > from the AWS Price List API entirely: querying `AmazonBedrock` returns only `gpt-oss` and GPT OSS Safeguard, and @@ -373,7 +380,8 @@ see `openai-on-bedrock.md`). GovCloud GPT-5.4 is 3.30 / 19.80. **Prompt caching (GPT-5.6 only):** cached input read at a 90% discount, cache write at 1.25x uncached input, minimum 1,024-token prefix, up to 4 breakpoints, ≥30-minute retention. Cached input is exempt from the input-TPM quota. -GPT-5.5 and GPT-5.4 do not list prompt caching support. +GPT-5.5 and GPT-5.4 list only implicit prompt caching on `bedrock-runtime` Responses (automatic, no request +parameters; cache-read rate above, no cache-write rate) and none on mantle (model cards read 2026-11-06). #### Open-weight gpt-oss diff --git a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json index a6bb34385..bed2a584b 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json @@ -3,8 +3,8 @@ "provider": "openai", "verified_at": "2026-09-24", "verified_region": "us-east-2", - "source": "Opus 5.5 cross-family target: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html (retrieved 2026-09-24). OpenAI and other unchanged entries retain their evidence from the superseded 2026-08-21 catalog (removed; see git history).", - "disclaimer": "The Opus cross-family target is updated to Opus 5.5; same-model OpenAI paths and their earlier evidence remain unchanged. This is a migration target catalog, not an account-availability claim. Verify the chosen model, API, region and CRIS profile before implementation. Opus 5.5 uses CRIS on runtime outside eu-west-2, has 1M context/128K output, always-on adaptive thinking, and no Batch tier. Fable and Mythos remain excluded from default selection. Opus 5.5 is the only Opus model in the automatic selection pool; older Opus snapshots and pricing records are historical references. The runtime in-region exception in eu-west-2 was verified 2026-10-03; the original catalog and other model evidence dates remain unchanged.", + "source": "Opus 5.5 cross-family target: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html (retrieved 2026-09-24). GPT-5.5 and GPT-5.4 rows re-read from their public model cards on 2026-11-06 (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html): both now list bedrock-runtime (Responses, Chat Completions, Converse via us./global. CRIS ids) and Chat Completions on bedrock-mantle. Other OpenAI entries retain their evidence from the superseded 2026-08-21 catalog (removed; see git history).", + "disclaimer": "The Opus cross-family target is updated to Opus 5.5; the GPT-5.5 and GPT-5.4 entries were re-read from their public model cards on 2026-11-06 (runtime CRIS paths, Chat Completions on mantle, 1.05M context, long-context pricing); the other same-model OpenAI paths and their earlier evidence remain unchanged. This is a migration target catalog, not an account-availability claim. Verify the chosen model, API, region and CRIS profile before implementation. Opus 5.5 uses CRIS on runtime outside eu-west-2, has 1M context/128K output, always-on adaptive thinking, and no Batch tier. Fable and Mythos remain excluded from default selection. Opus 5.5 is the only Opus model in the automatic selection pool; older Opus snapshots and pricing records are historical references. The runtime in-region exception in eu-west-2 was verified 2026-10-03; the original catalog and other model evidence dates remain unchanged.", "paths": { "mantle_openai_responses": { "sdk": "openai", @@ -19,8 +19,8 @@ "GPT-5.x on Bedrock Mantle is served through the Responses API" ], "constraints": [ - "Mantle itself has no cross-region form for any GPT model (in-region only); for GPT-5.6 the bedrock-runtime CRIS path covers additional regions instead (see runtime_converse)", - "GPT-5.x on Mantle rejects Chat Completions; a Chat Completions source must reshape to Responses (reference-verified on gpt-5.4; the 5.6 cards list Chat Completions as a supported API but its mantle-side availability is unprobed — verify in the target account)", + "Mantle itself has no cross-region form for any GPT model (in-region only); for GPT-5.6, GPT-5.5 and GPT-5.4 the bedrock-runtime CRIS path covers additional regions instead (see runtime_converse)", + "Chat Completions support on Mantle is model-specific: the GPT-5.4 and GPT-5.5 model cards list Chat Completions as supported on bedrock-mantle (read 2026-11-06); the GPT-5.6 Sol card does not. A Chat Completions source reshapes to Responses only when the selected model's card does not list Chat Completions — verify in the target account", "Availability must be probed in the target account and region" ] }, @@ -36,7 +36,7 @@ "OpenAI-compatible Chat Completions surface for models that accept it" ], "constraints": [ - "Not available for the GPT-5.x models listed here: rejected on gpt-5.4 (internal reference probe) and unsupported on the gpt-5.6-sol model card", + "Model-specific: listed as supported on bedrock-mantle by the GPT-5.4 and GPT-5.5 model cards (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, read 2026-11-06) and by the GPT-6 cards; unsupported on the gpt-5.6-sol model card. Uses /openai/v1/chat/completions", "Availability and model support must be probed in the target account and region" ] }, @@ -51,10 +51,10 @@ "strengths": [ "Bedrock-native multi-model request surface", "Supports Bedrock Guardrails (Converse API only), invocation logging, and CloudWatch", - "For GPT-5.6 this is a SAME-MODEL governance path via CRIS (us./in./global. ids; base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs); the model cards recommend runtime for new applications, and Global CRIS is priced at OpenAI's standard rate (cost parity)" + "For GPT-5.6, GPT-5.5 and GPT-5.4 this is a SAME-MODEL governance path via CRIS (us./in./global. ids; base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs); the model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" ], "constraints": [ - "GPT-5.5 / GPT-5.4 have NO runtime path — a Converse target for them means a Bedrock-native model and a boto3 rewrite of the OpenAI integration", + "GPT-5.5 / GPT-5.4 on runtime are CRIS-only (us. and global. ids; no in-region form) per their model cards (read 2026-11-06); the cards publish In-Region (1.10x) pricing only, so do not assume Global CRIS price parity for them", "GPT-5.6 on runtime is CRIS-only (no in-region form); Geo CRIS stays within the geography, Global CRIS routes anywhere — pick by data-residency posture. Runtime does not support server-side tool use, structured outputs, or application inference profiles for these models; prompt caching is Responses-only", "Bedrock-native models generally require a Global or geography-scoped CRIS inference profile", "Availability must be probed in the target account and region" @@ -111,19 +111,47 @@ "generation": "reasoning", "version": "5.5", "class": "reasoning", - "context_window": 272000, - "output_token_ceiling": "unknown", + "context_window": 1050000, + "output_token_ceiling": 128000, + "inference_profiles": { + "us": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1"], + "global": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1", "mx-central-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ap-southeast-5", "ap-southeast-6", "ap-southeast-7", "il-central-1", "me-central-1", "me-south-1", "af-south-1", "sa-east-1"] + }, + "region_availability": { + "commercial": ["us-east-1", "us-east-2"], + "govcloud": [], + "evidence": "bedrock-mantle in-region rows of the GPT-5.5 model card, read 2026-11-06: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html" + }, + "pricing": { + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-06); USD per 1M tokens, Standard tier; the card lists the In-Region inference option only", + "commercial_short_context_le_272k": {"input": 5.50, "cache_read": 0.55, "output": 33.00}, + "commercial_long_context_gt_272k": {"input": 11.00, "cache_read": 1.10, "output": 49.50}, + "service_tiers": {"standard": true, "priority": false, "flex": false, "reserved": false} + }, "capabilities": [ "reasoning", - "tool_or_function_calling" + "tool_or_function_calling", + "image_input_vision" ], - "capability_source": "AWS model card 2026-07-21 (launch 2026-06-01, Active): advanced coding/agentic tasks; tool calling via the Responses API's built-in tool use. Feature-level Mantle behavior not probed — verify before relying. Re-verified 2026-08-21: still mantle-only, in-region only; no CRIS, no runtime path, no long-context tier.", + "capability_source": "AWS model card https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-06; launch 2026-06-01, Active, EOL no sooner than 2027-06-01): text+image input, text output; 1,050,000-token context shared between input and output, 128,000 max output. Mantle lists server-side and client-side tool calling; runtime lists invocation logs, streaming, Guardrails (Converse only), application inference profiles (Converse only), implicit prompt caching (Responses only), structured outputs (Responses) and reasoning summaries; server-side tool use is unsupported on runtime. Documentation claims, not account probes — verify before relying.", "paths": { "mantle_openai_responses": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": false, - "evidence": "AWS model card model-card-openai-gpt-55 (2026-07-21): bedrock-mantle Programmatic Access with Responses sample code; Geo/Global inference not supported" + "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1 and us-east-2 only; Geo/Global not supported on mantle" + }, + "mantle_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.5", + "requires_cris": false, + "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code" + }, + "runtime_converse": { + "available": true, + "model_id": "openai.gpt-5.5", + "requires_cris": true, + "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.5 (Geo) or global.openai.gpt-5.5 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs" } } }, @@ -213,27 +241,50 @@ "generation": "reasoning", "version": "5.4", "class": "reasoning", - "context_window": "unknown", - "output_token_ceiling": "unknown", + "context_window": 1050000, + "output_token_ceiling": 128000, + "inference_profiles": { + "us": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1"], + "global": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1", "mx-central-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ap-southeast-5", "ap-southeast-6", "ap-southeast-7", "il-central-1", "me-central-1", "me-south-1", "af-south-1", "sa-east-1"] + }, + "region_availability": { + "commercial": ["us-east-1", "us-east-2", "us-west-2"], + "govcloud": ["us-gov-west-1"], + "evidence": "bedrock-mantle in-region rows of the GPT-5.4 model card, read 2026-11-06: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html" + }, + "pricing": { + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-06); USD per 1M tokens, Standard tier; the card lists the In-Region inference option only", + "commercial_short_context_le_272k": {"input": 2.75, "cache_read": 0.275, "output": 16.50}, + "commercial_long_context_gt_272k": {"input": 5.50, "cache_read": 0.55, "output": 24.75}, + "govcloud_us_west_short_context_le_272k": {"input": 3.375, "cache_read": 0.3375, "output": 20.25}, + "service_tiers": {"standard": true, "priority": false, "flex": false, "reserved": false} + }, "capabilities": [ "tool_or_function_calling", "structured_output_json", "reasoning", - "streaming" + "streaming", + "image_input_vision" ], - "capability_source": "internal reference smoke tests and its reasoning-API migration notebook (Responses path, 2026-06 probes) — the only entry with feature-level Mantle behavior probes (tools, structured output, sampling); limits not stated Re-verified 2026-08-21: still mantle-only, in-region only; no CRIS, no runtime path, no long-context tier.", + "capability_source": "AWS model card https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-06; launch 2026-06-01, Active, EOL no sooner than 2027-06-01): text+image input, text output; 1,050,000-token context shared between input and output, 128,000 max output. Mantle lists server-side and client-side tool calling; runtime lists invocation logs, response streaming, Guardrails (Converse only), application inference profiles (Converse only), implicit prompt caching (Responses only), structured outputs (JSON Schema; Responses API) and reasoning summaries; server-side tool use is unsupported on runtime. Documentation claims, not account probes — verify before relying.", "paths": { "mantle_openai_responses": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": false, - "evidence": "internal reference smoke-test default Mantle model id; client.responses.create verified live" + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1, us-east-2, us-west-2 and us-gov-west-1; Geo/Global not supported on mantle" }, "mantle_openai_chat": { - "available": false, + "available": true, "model_id": "openai.gpt-5.4", "requires_cris": false, - "evidence": "internal reference verified gotcha: GPT-5.x on Mantle is Responses-only; Chat Completions rejected" + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code" + }, + "runtime_converse": { + "available": true, + "model_id": "openai.gpt-5.4", + "requires_cris": true, + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.4 (Geo) or global.openai.gpt-5.4 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs" } } }, diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md index cdf8ced55..7720d4b58 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md @@ -190,12 +190,18 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` ## migration_path vocabulary `mantle_openai_responses` (an OpenAI-protocol source — direct or Azure-fronted — with the model -on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `converse` -(Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when a proprietary GPT -model must move off mantle for Guardrails / Knowledge Bases / logging / an unsupported region) · -`gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic -Bedrock SDK swap). A proprietary `openai.gpt-*` model ID is NEVER paired with a -`converse`/`bedrock-runtime` path — those models are mantle-only. +on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `mantle_openai_chat` +(same, preserving a Chat Completions source on a model whose card lists Chat Completions on +mantle — GPT-6, GPT-5.5, GPT-5.4) · `runtime_openai_cris` (same-model move served via +`bedrock-runtime` with a `us.`/`in.`/`global.` CRIS id, `model_change: false`; available for +GPT-6, GPT-5.6, GPT-5.5 and GPT-5.4 per their model cards) · `converse` (Bedrock-native +Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, +intelligent prompt routing, or another feature no same-model path serves) · `gpt-oss` +(OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK +swap). A **bare** proprietary `openai.gpt-*` model ID is NEVER paired with a +`converse`/`bedrock-runtime` path — bare ids are mantle ids; every proprietary GPT target on +`bedrock-runtime` carries a CRIS-prefixed id (there is no runtime in-region form for GPT-6, +GPT-5.6, GPT-5.5 or GPT-5.4). ## regional_validation @@ -255,8 +261,9 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` `price_comparison`. - [ ] `honest_assessment` overall is the weakest across `bedrock_models[]`; if `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. -- [ ] No `bedrock_models[]` entry pairs a proprietary `openai.gpt-*` model ID with a - `converse`/`bedrock-runtime` migration path (mantle-only). +- [ ] No `bedrock_models[]` entry pairs a **bare** proprietary `openai.gpt-*` model ID with a + `converse`/`bedrock-runtime` migration path (bare ids are mantle ids; a `bedrock-runtime` + target for GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 carries a `us.`/`in.`/`global.` CRIS id). - [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves to direct OpenAI: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. diff --git a/plugins/aws-startup-advisor/tests/test_gpt6_review_round2.py b/plugins/aws-startup-advisor/tests/test_gpt6_review_round2.py index 7af115742..2309b54cb 100644 --- a/plugins/aws-startup-advisor/tests/test_gpt6_review_round2.py +++ b/plugins/aws-startup-advisor/tests/test_gpt6_review_round2.py @@ -70,15 +70,25 @@ def test_both_estimate_ai_files_require_matching_context_tiers() -> None: # --- Finding 2: runtime_openai_cris gate/dispatch --- -def test_design_ai_scopes_mantle_only_to_gpt55_gpt54() -> None: +def test_design_ai_gives_gpt55_gpt54_the_runtime_cris_path() -> None: + # GPT-5.5 / GPT-5.4 model cards (read 2026-11-06) list us./global. CRIS ids on + # bedrock-runtime, so the runtime_openai_cris carve-out must name them alongside + # GPT-6/5.6 and the old "mantle-only, no CRIS" scoping must be gone. for f in DESIGN_AI_FILES: - text = f.read_text(encoding="utf-8") + text = re.sub(r"\s+", " ", f.read_text(encoding="utf-8")) assert "runtime_openai_cris" in text, ( - f"{f}: no runtime_openai_cris carve-out for GPT-6/5.6 (Finding 2)" + f"{f}: no runtime_openai_cris carve-out (Finding 2)" ) assert "GPT-5.5" in text and "GPT-5.4" in text, ( - f"{f}: mantle-only scoping must name GPT-5.5/GPT-5.4 specifically (Finding 2)" + f"{f}: the runtime_openai_cris rule must name GPT-5.5/GPT-5.4 specifically" + ) + assert "us.openai.gpt-5.5" in text and "us.openai.gpt-5.4" in text, ( + f"{f}: must record the GPT-5.5/GPT-5.4 us. CRIS ids" ) + assert "GPT-5.5 and GPT-5.4 are mantle-only" not in text + assert "GPT-5.5 and GPT-5.4 are `bedrock-mantle` only" not in text + assert "no CRIS" not in text + assert "No Converse fallback for GPT-5.5/GPT-5.4" not in text def test_generate_artifacts_ai_dispatches_runtime_openai_cris() -> None: @@ -102,9 +112,16 @@ def test_schema_design_aws_ai_vocabulary_includes_runtime_openai_cris() -> None: "schema-design-aws-ai.md: migration_path vocabulary missing runtime_openai_cris (Finding 2)" ) assert "GPT-5.5" in text and "GPT-5.4" in text, ( - "schema-design-aws-ai.md: mantle-only scoping must name GPT-5.5/GPT-5.4 specifically, " - "not blanket 'proprietary GPT' (Finding 2)" + "schema-design-aws-ai.md: runtime_openai_cris vocabulary must name GPT-5.5/GPT-5.4 " + "specifically, not blanket 'proprietary GPT' (Finding 2)" ) + # The ban is on BARE proprietary GPT ids paired with converse/bedrock-runtime, + # not on the GPT-5.5/GPT-5.4 models — their cards list us./global. CRIS ids. + normalized = re.sub(r"\s+", " ", text) + assert "those models are mantle-only" not in normalized + assert "mantle-only, no CRIS" not in normalized + assert "**bare** proprietary `openai.gpt-*`" in normalized + assert "us.openai.gpt-5.5" in normalized or "GPT-5.5 / GPT-5.4 runtime support" in normalized # --- Finding 3: Astra Invoke-support correction --- diff --git a/plugins/aws-startup-advisor/tests/test_gpt6_review_round5.py b/plugins/aws-startup-advisor/tests/test_gpt6_review_round5.py index c73b08235..187601bd6 100644 --- a/plugins/aws-startup-advisor/tests/test_gpt6_review_round5.py +++ b/plugins/aws-startup-advisor/tests/test_gpt6_review_round5.py @@ -114,6 +114,12 @@ def test_behavior_delta_no_longer_claims_chat_completions_unverified() -> None: assert "OPTIONAL" in text, ( "openai-to-bedrock.md must make the Responses reshape optional, not mandatory" ) + # GPT-5.5 / GPT-5.4 model cards (read 2026-11-06) list Chat Completions on both + # bedrock-mantle and bedrock-runtime — the reshape is optional for them too. + assert "GPT-5.5 / GPT-5.4 follow the same rule" in text + assert "model-card-openai-gpt-54.html" in text and "model-card-openai-gpt-55.html" in text + assert "mantle-only, no CRIS" not in text + assert "still unconfirmed" not in text def test_both_design_ai_no_longer_force_gpt6_chat_completions_reshape() -> None: @@ -122,12 +128,17 @@ def test_both_design_ai_no_longer_force_gpt6_chat_completions_reshape() -> None: assert "near-drop-in" in text.lower() or "optional" in text.lower(), ( f"{f}: a GPT-6/GPT-5.6 Chat Completions source must not be forced to reshape" ) - assert "GPT-5.5" in text and "GPT-5.4" in text, ( - f"{f}: the mandatory-reshape exception must still be scoped to GPT-5.5/GPT-5.4" + assert "GPT-5.5 and GPT-5.4 follow the same rule" in text, ( + f"{f}: GPT-5.5/GPT-5.4 Chat Completions is listed on both endpoints (cards read " + "2026-11-06); they must not be the mandatory-reshape exception" ) + assert "GPT-5.5 and GPT-5.4 are the exception" not in text + assert "unconfirmed" not in text def test_gcp_generate_ai_no_longer_forces_gpt6_chat_completions_reshape() -> None: text = _norm(GCP_GENERATE_AI_MD.read_text(encoding="utf-8")) assert "optional" in text.lower() - assert "GPT-5.5" in text and "GPT-5.4" in text + assert "GPT-5.5/GPT-5.4 follow the same rule" in text + assert "still need a reshape" not in text + assert "unconfirmed" not in text diff --git a/plugins/aws-startup-advisor/tests/test_gpt6_round4_findings.py b/plugins/aws-startup-advisor/tests/test_gpt6_round4_findings.py index f375e2ef2..46e2ac3a8 100644 --- a/plugins/aws-startup-advisor/tests/test_gpt6_round4_findings.py +++ b/plugins/aws-startup-advisor/tests/test_gpt6_round4_findings.py @@ -2,8 +2,9 @@ GPT-6 recognition, report pricing bugs). Finding 0 — runtime_openai_cris carve-out gaps: - (a) Azure design.md / generate.md _assert blocks must scope the GPT-ban to - GPT-5.5/GPT-5.4 and explicitly carve out GPT-5.6/GPT-6 on runtime_openai_cris. + (a) Azure design.md / generate.md _assert blocks must scope the GPT-ban to BARE + proprietary ids and explicitly carve out GPT-5.6/GPT-6 — and, since the + 2026-11-06 model-card re-read, GPT-5.5/GPT-5.4 — on runtime_openai_cris. (b) GCP generate-ai.md Part 1 must route runtime_openai_cris through bedrock-runtime, not the mantle URL/IAM. (c) Canonical model-selection guide + vendored guardrails must name the full @@ -79,21 +80,30 @@ def _norm(text: str) -> str: # --- Finding 0.1: Azure design.md / generate.md assert carve-out --- -def test_azure_design_assert_scopes_gpt_ban_to_55_54_and_carves_out_cris() -> None: +def test_azure_design_assert_scopes_gpt_ban_to_bare_ids_and_carves_out_cris() -> None: + # GPT-5.5 / GPT-5.4 model cards (read 2026-11-06) list us./global. CRIS ids on + # bedrock-runtime, so the ban is on BARE proprietary ids paired with + # converse/bedrock-runtime — not on the GPT-5.5/GPT-5.4 models themselves. text = _norm(AZURE_DESIGN_MD.read_text(encoding="utf-8")) - assert "no proprietary GPT-5.5 or GPT-5.4 model ID is paired" in text, ( - "design.md _assert must scope the ban to GPT-5.5/GPT-5.4, not blanket 'openai.gpt-*'" + assert "no bare proprietary openai.gpt-* model ID (including bare openai.gpt-5.5 or openai.gpt-5.4) is paired" in text, ( + "design.md _assert must ban bare proprietary ids on converse/bedrock-runtime, " + "naming bare openai.gpt-5.5/openai.gpt-5.4 explicitly" ) - assert "runtime_openai_cris on bedrock-runtime — this is not a violation" in text + assert "GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 models MAY use migration_path: runtime_openai_cris on bedrock-runtime — this is not a violation" in text + assert "GPT-5.5/GPT-5.4 us./global. profiles" in text + assert "no proprietary GPT-5.5 or GPT-5.4 model ID is paired" not in text assert "no proprietary openai.gpt-* model ID is paired" not in text -def test_azure_generate_assert_scopes_gpt_ban_to_55_54_and_carves_out_cris() -> None: +def test_azure_generate_assert_scopes_gpt_ban_to_bare_ids_and_carves_out_cris() -> None: text = _norm(AZURE_GENERATE_MD.read_text(encoding="utf-8")) - assert "no proprietary GPT-5.5 or GPT-5.4 model ID is paired" in text, ( - "generate.md _assert must scope the ban to GPT-5.5/GPT-5.4, not blanket 'openai.gpt-*'" + assert "no bare proprietary openai.gpt-* model ID (including bare openai.gpt-5.5 or openai.gpt-5.4) is paired" in text, ( + "generate.md _assert must ban bare proprietary ids on converse/bedrock-runtime, " + "naming bare openai.gpt-5.5/openai.gpt-5.4 explicitly" ) - assert "runtime_openai_cris on bedrock-runtime — this is not a violation" in text + assert "GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 models MAY use migration_path: runtime_openai_cris on bedrock-runtime — this is not a violation" in text + assert "GPT-5.5/GPT-5.4 us./global. profiles" in text + assert "no proprietary GPT-5.5 or GPT-5.4 model ID is paired" not in text assert "no proprietary openai.gpt-* model ID is paired" not in text From e72d05df801a77c7a8987b708e304d98713b51fa Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Fri, 6 Nov 2026 23:35:29 -0800 Subject: [PATCH 02/41] fix(aws-startup-advisor): pin and disclose the GPT-6 Chat fallback shift caused by the GPT-5.4/5.5 catalog facts Review round 1 (CHANGES_REQUESTED, 2 blockers) on c1944a53. Finding 1: recording the public GPT-5.5/GPT-5.4 facts in skills/shared/ai/models/openai-bedrock.json (mantle_openai_chat available, region_availability) changes agent-advisor recommendations through the unchanged catalog-order fallback in openai_model_recommendation.py (_mantle_candidate_order -> _catalog_model_for_path): GPT-5.5 and GPT-5.4 precede GPT-6 Astra in catalog key order, so a GPT-6 Sol / 6.1 Sol Chat Completions source outside us-east-1 now lands on - openai.gpt-5.5 via mantle_openai_chat in us-east-2 (base: decision_required, mantle_region_unavailable, options mantle chat / runtime_converse) - openai.gpt-5.4 via mantle_openai_chat in us-west-2 (base: openai.gpt-6-astra) and a GPT-5.6 Sol Responses source in us-west-2 lands on openai.gpt-5.6-terra (base: openai.gpt-5.5, whose cataloged footprint is us-east-1/us-east-2 only). The previous commit message called this PR "no behavior change"; that was wrong for these cells. The catalog facts are kept (they are what the public cards state); the engine ordering is NOT changed in this PR. Instead: - skills/agent-advisor/scripts/test_openai_model_recommendation.py: new regression tests pin all five cells so the shift is deliberate and visible (test_gpt6_chat_outside_own_footprint_falls_back_to_in_region_gpt5_chat x4, test_gpt56_sol_responses_outside_us_east_skips_gpt55_for_in_region_terra). - skills/agent-advisor/references/phases/model-recommend/model-recommend.md: the engine-limitation paragraph now states that Chat fallbacks walk catalog order, that GPT-5.5/5.4 precede GPT-6 Astra, and that a GPT-6 Chat source outside us-east-1 lands on GPT-5.5 (us-east-2) / GPT-5.4 (us-west-2); a newer-generation source falling back to an older-generation model is an engine ordering limitation to resolve in the follow-up selection PR (PR 2). Finding 2: skills/agent-advisor/scripts/openai_model_recommendation.py:118-119 still said "the (Mantle-only) GPT-5.x family". Reworded to the engine fact (GPT-5.x sources not recognized as same-model runtime identities map to Claude tiers; GPT-5.5/5.4 do carry a cataloged runtime_converse path via us./global. CRIS ids). Grep over plugins/ for mantle-only / "no CRIS" / "no inference profile" / unconfirmed / "internal reference" near GPT-5.4/5.5/5.x is now clean apart from negative test assertions, the GovCloud Opus 5.5 pricing row, and the bare-id rule ("bare proprietary openai.gpt-* IDs use Mantle only"). Carry-forward for PR 2 (endpoint selection): reorder / tier the mantle_openai_chat candidate walk so a GPT-6 source does not fall back to GPT-5.4/5.5 ahead of GPT-6 Astra; add GPT-5.5/5.4 to _SOURCE_MODEL_KEYS so they are recognized as same-model runtime identities. No new behavior added; no dispatch or endpoint-selection code changed. No vendored files affected (none of the three files is in the sync manifest); lint:vendored-parity re-run and still 62/62 byte-identical. Sources (public model cards, read 2026-11-06 via the AWS Documentation MCP tools): - https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html - https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html Validation (run from the worktree with mise Python 3.12): - mise -C plugins/aws-startup-advisor run ci: PASS - lint:md 0 errors; lint:model-id OK (19 retired IDs enforced); lint:model-staleness fresh (4d old, 45d window); lint:model-catalog-drift OK (3 pre-existing warnings, unrelated); lint:startup-model-pricing OK; lint:artifacts PASS (99 checked, 0 errors); lint:vendored-parity PASS (62 files byte-identical); lint:skill-refs PASS (1482 checked, 0 missing); lint:consent-disclosure OK (6 checked); test: 1395 passed, 222 subtests passed; test:telemetry 311 passed; test:heroku-source-review 44 pass, 0 fail - agent-advisor pytest (skills/agent-advisor/scripts): 569 passed, 0 failed (564 at c1944a53 + 5 new) - llm-to-bedrock pytest (skills/llm-to-bedrock/scripts): 418 passed, 0 failed - root: python3 tools/validate.py "All validations passed"; mise run lint:md 0 issues / 2763 files; lint:manifests, lint:spec, lint:skill-parity pass; test:tools OK. mise run build's security step still fails only on the pre-existing gitleaks finding in upstream commit f8d09fd1 (tests/test_bedrock_migration_assessment_discover.py:118); gitleaks protect --staged over this change set: no leaks found. Base: 0a218f8cc13220013f72147e2e0fdbe350b34ae2 (upstream/main) --- .../phases/model-recommend/model-recommend.md | 9 +++- .../scripts/openai_model_recommendation.py | 5 +- .../test_openai_model_recommendation.py | 51 +++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md index c418842ed..013ceaf50 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md @@ -104,7 +104,14 @@ use `mantle_openai_responses`. Both Mantle paths are in-region only and enforce the selected model's cataloged `region_availability`: Astra reaches us-east-1 and us-west-2, while Sol, Luna, and 6.1 Sol reach us-east-1 only. Outside that footprint, the engine skips the same-model Mantle candidate and evaluates its existing capability/numeric -fallbacks. Governance requirements select the exact same GPT-5.6 or GPT-6 model through +fallbacks. Those fallbacks walk candidates in catalog order, and GPT-5.5 and GPT-5.4 (both +cataloged with Chat Completions on Mantle; GPT-5.5 reaches us-east-1/us-east-2, GPT-5.4 reaches +us-east-1/us-east-2/us-west-2 and us-gov-west-1) precede GPT-6 Astra in that order. A GPT-6 Sol or +6.1 Sol Chat Completions source outside us-east-1 therefore lands on GPT-5.5 in us-east-2 or GPT-5.4 +in us-west-2 through `mantle_openai_chat`, not on Astra. That is a current engine ordering +limitation (a newer-generation source falls back to an older-generation model); treat such a +recommendation as a prompt for the Chat-vs-Converse decision, not as a model endorsement. +Governance requirements select the exact same GPT-5.6 or GPT-6 model through `runtime_converse` only when the requested Global or geography-scoped CRIS profile is cataloged for that model and source region; an unsupported profile falls through to an eligible documented runtime candidate or remains an explicit decision. GPT-5.5/5.4 also carry a cataloged diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py index e3b1a07e0..47aeed856 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py @@ -116,7 +116,10 @@ def _delta(code, category, description): # Converse tier map: a governance workload keeps its capability tier when moving -# from the (Mantle-only) GPT-5.x family to Claude on runtime Converse. +# from a GPT-5.x source the engine does not recognize as a same-model runtime +# identity (see _SOURCE_MODEL_KEYS) to Claude on runtime Converse. This is an +# engine recognition gap, not an endpoint fact: GPT-5.5 and GPT-5.4 do carry a +# cataloged runtime_converse path via us./global. CRIS ids. # GPT-5.6 Sol (frontier) -> Claude Opus 5.5 # GPT-5.6 Terra / 5.5 / 5.4 -> Claude Sonnet 5 (also the default tier) # GPT-5.6 Luna (fast/low-cost)-> Claude Haiku 4.5 diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py index 7d22b84a7..0ebb2a29a 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py @@ -308,6 +308,57 @@ def test_gpt6_mantle_skips_exact_model_outside_cataloged_region(): assert "exact same model" not in rec["rationale"][0] +# Deliberate consequence of recording the public GPT-5.5/GPT-5.4 facts (Chat +# Completions available on Mantle, cataloged region_availability): the Mantle Chat +# fallback walks candidates in catalog order, and GPT-5.5 / GPT-5.4 sit ahead of +# GPT-6 Astra in that order. A GPT-6 Sol / 6.1 Sol Chat source outside its own +# footprint therefore lands on GPT-5.5 (us-east-2) or GPT-5.4 (us-west-2) instead +# of Astra / a blocking decision. Pinned here so the shift is visible; reordering +# the candidate walk is engine work tracked for the follow-up PR. +@pytest.mark.parametrize("source_id", ["gpt-6-sol", "gpt-6.1-sol"]) +@pytest.mark.parametrize( + ("region", "fallback_id", "fallback_key"), + [ + ("us-east-2", "openai.gpt-5.5", "openai_gpt_5_5"), + ("us-west-2", "openai.gpt-5.4", "openai_gpt_5_4"), + ], +) +def test_gpt6_chat_outside_own_footprint_falls_back_to_in_region_gpt5_chat( + source_id, region, fallback_id, fallback_key +): + rec = _recommend( + _workload( + source={"model_ids": [source_id], "api_surface": "chat_completions"}, + requirements={"api_continuity": "required"}, + ), + region=region, + ) + assert rec["decision_status"] == "recommended" + assert rec["primary_model"] == fallback_id + assert rec["api_path"] == "mantle_openai_chat" + assert "mantle_region_unavailable" not in _codes(rec["blocks"]) + assert "chat_completions_to_responses_required" not in _codes(rec["blocks"]) + assert "fallback" in rec["rationale"][0] + assert region in OPENAI_CATALOG["models"][fallback_key]["region_availability"]["commercial"] + + +def test_gpt56_sol_responses_outside_us_east_skips_gpt55_for_in_region_terra(): + # GPT-5.5's cataloged footprint is us-east-1/us-east-2 only, so in us-west-2 + # the Responses fallback now skips it and lands on GPT-5.6 Terra (base + # catalog, with no GPT-5.5 region data, selected GPT-5.5 here). + rec = _recommend( + _workload( + source={"model_ids": ["gpt-5.6-sol"]}, + requirements={"api_continuity": "required"}, + ), + region="us-west-2", + ) + assert rec["decision_status"] == "recommended" + assert rec["primary_model"] == "openai.gpt-5.6-terra" + assert rec["api_path"] == "mantle_openai_responses" + assert "us-west-2" not in OPENAI_CATALOG["models"]["openai_gpt_5_5"]["region_availability"]["commercial"] + + def test_gpt6_chat_outside_every_mantle_footprint_returns_blocking_decision(): workload = _workload( source={"model_ids": ["gpt-6-sol"], "api_surface": "chat_completions"}, From bc51f0cc28406296db46d6ed81ee5941308d486d Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 7 Nov 2026 08:52:30 -0800 Subject: [PATCH 03/41] fix(aws-startup-advisor): scope the in. CRIS prefix and runtime feature constraints per model for GPT-5.4/5.5 The GPT-5.4 and GPT-5.5 model cards publish only us. and global. CRIS ids; the in. profile exists for GPT-5.6 only. The runtime_converse path strength and both schema-design vocabulary/checklist lines attached us./in./global. to all three models, contradicting the catalog's own next constraint and openai-on-bedrock.md. Rewritten so in. stays on GPT-5.6 / GPT-6. The GPT-5.6 runtime constraint ("does not support server-side tool use, structured outputs, or application inference profiles") is now scoped to GPT-5.6, and a separate GPT-5.4/5.5 constraint records what their cards mark: server-side tools unsupported; structured outputs and implicit prompt caching Responses-only; Guardrails and application inference profiles Converse-only. Vendored copies re-synced (62 checked, 3 synced). Regression test added. --- .../vendored/ai/models/openai-bedrock.json | 5 +++-- .../test_openai_model_recommendation.py | 18 ++++++++++++++++++ .../references/shared/schema-design-aws-ai.md | 7 ++++--- .../vendored/ai/schema-design-aws-ai.md | 7 ++++--- .../vendored/ai/schema-design-aws-ai.md | 7 ++++--- .../shared/ai/models/openai-bedrock.json | 5 +++-- .../skills/shared/ai/schema-design-aws-ai.md | 7 ++++--- 7 files changed, 40 insertions(+), 16 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json index bed2a584b..48758f3c9 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json @@ -51,11 +51,12 @@ "strengths": [ "Bedrock-native multi-model request surface", "Supports Bedrock Guardrails (Converse API only), invocation logging, and CloudWatch", - "For GPT-5.6, GPT-5.5 and GPT-5.4 this is a SAME-MODEL governance path via CRIS (us./in./global. ids; base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs); the model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" + "For GPT-5.6 this is a SAME-MODEL governance path via CRIS (us./in./global. ids). For GPT-5.5 and GPT-5.4 the runtime ids are us. and global. only (their model cards publish no in. profile). The base URL for the OpenAI-compatible APIs is bedrock-runtime.{region}.amazonaws.com/openai/v1. The model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" ], "constraints": [ "GPT-5.5 / GPT-5.4 on runtime are CRIS-only (us. and global. ids; no in-region form) per their model cards (read 2026-11-06); the cards publish In-Region (1.10x) pricing only, so do not assume Global CRIS price parity for them", - "GPT-5.6 on runtime is CRIS-only (no in-region form); Geo CRIS stays within the geography, Global CRIS routes anywhere — pick by data-residency posture. Runtime does not support server-side tool use, structured outputs, or application inference profiles for these models; prompt caching is Responses-only", + "GPT-5.6 on runtime is CRIS-only (no in-region form); Geo CRIS stays within the geography, Global CRIS routes anywhere — pick by data-residency posture. For GPT-5.6, runtime does not support server-side tool use, structured outputs, or application inference profiles; prompt caching is Responses-only", + "GPT-5.5 / GPT-5.4 on runtime (per their model cards, read 2026-11-07): server-side tool use is unsupported; structured outputs and implicit prompt caching are Responses-only; Guardrails and application inference profiles are Converse-only", "Bedrock-native models generally require a Global or geography-scoped CRIS inference profile", "Availability must be probed in the target account and region" ] diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py index 0ebb2a29a..c506f6830 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py @@ -116,6 +116,24 @@ def test_catalog_gpt54_gpt55_record_public_runtime_and_chat_facts(catalog_key, m assert "internal reference" not in model["capability_source"] +def test_runtime_converse_path_text_does_not_grant_in_profile_to_gpt54_gpt55(): + # The GPT-5.4 / GPT-5.5 cards publish only us. and global. CRIS ids; only + # GPT-5.6 has an in. profile. The shared runtime_converse path text must not + # attach the in. prefix to the 5.4/5.5 models, and its feature constraints + # must be stated per model (5.4/5.5 cards list structured outputs on Responses + # and application inference profiles on Converse, which the GPT-5.6 sentence + # denies). + path = OPENAI_CATALOG["paths"]["runtime_converse"] + strengths = "\n".join(path["strengths"]) + constraints = "\n".join(path["constraints"]) + assert "For GPT-5.6, GPT-5.5 and GPT-5.4" not in strengths + assert "GPT-5.5 and GPT-5.4 the runtime ids are us. and global. only" in strengths + assert "For GPT-5.6, runtime does not support server-side tool use" in constraints + assert "GPT-5.5 / GPT-5.4 on runtime" in constraints + assert "structured outputs and implicit prompt caching are Responses-only" in constraints + assert "Guardrails and application inference profiles are Converse-only" in constraints + + def test_chat_completions_case_includes_full_reshape_deltas(): rec = _recommend( _workload( diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md index 36a049656..9d5c929aa 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md @@ -127,8 +127,8 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` lists it there (GPT-6, GPT-5.5, GPT-5.4). `mantle_openai_responses` (Azure OpenAI / OpenAI source, model on Bedrock, region carries it — keep the SDK, `model_change: false`) · `runtime_openai_cris` (GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 -same-model move served via `bedrock-runtime` CRIS, `us.`/`in.`/`global.` prefixed, -`model_change: false`) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set +same-model move served via `bedrock-runtime` CRIS — `us.`/`in.`/`global.` prefixed for GPT-6 / +GPT-5.6, `us.`/`global.` only for GPT-5.5 / GPT-5.4 — `model_change: false`) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, intelligent prompt routing, or another feature the same-model CRIS path itself cannot serve) · `gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK swap). A **bare** @@ -207,7 +207,8 @@ rubric file (`ai.md`, `ai-azure-openai-to-bedrock.md`, or a `vendored/ai/*` ref) - [ ] No `bedrock_models[]` entry pairs a **bare** proprietary GPT id (including bare `openai.gpt-5.5` / `openai.gpt-5.4`) with a `converse`/`bedrock-runtime` migration path. GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 MAY use `migration_path: "runtime_openai_cris"` on - `bedrock-runtime` with a `us.`/`in.`/`global.` CRIS id — this is NOT a violation. + `bedrock-runtime` with a CRIS id (`us.`/`in.`/`global.` for GPT-6 / GPT-5.6, `us.`/`global.` + only for GPT-5.5 / GPT-5.4) — this is NOT a violation. - [ ] If `ai_source` is `azure_openai` or `openai`: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md index 7720d4b58..c31359bbc 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -193,8 +193,8 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `mantle_openai_chat` (same, preserving a Chat Completions source on a model whose card lists Chat Completions on mantle — GPT-6, GPT-5.5, GPT-5.4) · `runtime_openai_cris` (same-model move served via -`bedrock-runtime` with a `us.`/`in.`/`global.` CRIS id, `model_change: false`; available for -GPT-6, GPT-5.6, GPT-5.5 and GPT-5.4 per their model cards) · `converse` (Bedrock-native +`bedrock-runtime` with a CRIS id, `model_change: false`; `us.`/`in.`/`global.` for GPT-6 and +GPT-5.6, `us.`/`global.` only for GPT-5.5 and GPT-5.4, per their model cards) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, intelligent prompt routing, or another feature no same-model path serves) · `gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK @@ -263,7 +263,8 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. - [ ] No `bedrock_models[]` entry pairs a **bare** proprietary `openai.gpt-*` model ID with a `converse`/`bedrock-runtime` migration path (bare ids are mantle ids; a `bedrock-runtime` - target for GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 carries a `us.`/`in.`/`global.` CRIS id). + target carries a CRIS id: `us.`/`in.`/`global.` for GPT-6 / GPT-5.6, `us.`/`global.` only for + GPT-5.5 / GPT-5.4). - [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves to direct OpenAI: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md index 7720d4b58..c31359bbc 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -193,8 +193,8 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `mantle_openai_chat` (same, preserving a Chat Completions source on a model whose card lists Chat Completions on mantle — GPT-6, GPT-5.5, GPT-5.4) · `runtime_openai_cris` (same-model move served via -`bedrock-runtime` with a `us.`/`in.`/`global.` CRIS id, `model_change: false`; available for -GPT-6, GPT-5.6, GPT-5.5 and GPT-5.4 per their model cards) · `converse` (Bedrock-native +`bedrock-runtime` with a CRIS id, `model_change: false`; `us.`/`in.`/`global.` for GPT-6 and +GPT-5.6, `us.`/`global.` only for GPT-5.5 and GPT-5.4, per their model cards) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, intelligent prompt routing, or another feature no same-model path serves) · `gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK @@ -263,7 +263,8 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. - [ ] No `bedrock_models[]` entry pairs a **bare** proprietary `openai.gpt-*` model ID with a `converse`/`bedrock-runtime` migration path (bare ids are mantle ids; a `bedrock-runtime` - target for GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 carries a `us.`/`in.`/`global.` CRIS id). + target carries a CRIS id: `us.`/`in.`/`global.` for GPT-6 / GPT-5.6, `us.`/`global.` only for + GPT-5.5 / GPT-5.4). - [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves to direct OpenAI: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json index bed2a584b..48758f3c9 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json @@ -51,11 +51,12 @@ "strengths": [ "Bedrock-native multi-model request surface", "Supports Bedrock Guardrails (Converse API only), invocation logging, and CloudWatch", - "For GPT-5.6, GPT-5.5 and GPT-5.4 this is a SAME-MODEL governance path via CRIS (us./in./global. ids; base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs); the model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" + "For GPT-5.6 this is a SAME-MODEL governance path via CRIS (us./in./global. ids). For GPT-5.5 and GPT-5.4 the runtime ids are us. and global. only (their model cards publish no in. profile). The base URL for the OpenAI-compatible APIs is bedrock-runtime.{region}.amazonaws.com/openai/v1. The model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" ], "constraints": [ "GPT-5.5 / GPT-5.4 on runtime are CRIS-only (us. and global. ids; no in-region form) per their model cards (read 2026-11-06); the cards publish In-Region (1.10x) pricing only, so do not assume Global CRIS price parity for them", - "GPT-5.6 on runtime is CRIS-only (no in-region form); Geo CRIS stays within the geography, Global CRIS routes anywhere — pick by data-residency posture. Runtime does not support server-side tool use, structured outputs, or application inference profiles for these models; prompt caching is Responses-only", + "GPT-5.6 on runtime is CRIS-only (no in-region form); Geo CRIS stays within the geography, Global CRIS routes anywhere — pick by data-residency posture. For GPT-5.6, runtime does not support server-side tool use, structured outputs, or application inference profiles; prompt caching is Responses-only", + "GPT-5.5 / GPT-5.4 on runtime (per their model cards, read 2026-11-07): server-side tool use is unsupported; structured outputs and implicit prompt caching are Responses-only; Guardrails and application inference profiles are Converse-only", "Bedrock-native models generally require a Global or geography-scoped CRIS inference profile", "Availability must be probed in the target account and region" ] diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md index 7720d4b58..c31359bbc 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md @@ -193,8 +193,8 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `mantle_openai_chat` (same, preserving a Chat Completions source on a model whose card lists Chat Completions on mantle — GPT-6, GPT-5.5, GPT-5.4) · `runtime_openai_cris` (same-model move served via -`bedrock-runtime` with a `us.`/`in.`/`global.` CRIS id, `model_change: false`; available for -GPT-6, GPT-5.6, GPT-5.5 and GPT-5.4 per their model cards) · `converse` (Bedrock-native +`bedrock-runtime` with a CRIS id, `model_change: false`; `us.`/`in.`/`global.` for GPT-6 and +GPT-5.6, `us.`/`global.` only for GPT-5.5 and GPT-5.4, per their model cards) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, intelligent prompt routing, or another feature no same-model path serves) · `gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK @@ -263,7 +263,8 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. - [ ] No `bedrock_models[]` entry pairs a **bare** proprietary `openai.gpt-*` model ID with a `converse`/`bedrock-runtime` migration path (bare ids are mantle ids; a `bedrock-runtime` - target for GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 carries a `us.`/`in.`/`global.` CRIS id). + target carries a CRIS id: `us.`/`in.`/`global.` for GPT-6 / GPT-5.6, `us.`/`global.` only for + GPT-5.5 / GPT-5.4). - [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves to direct OpenAI: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. From dfd07bdcd352babffca2e2da32fce5661f06b8df Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 7 Nov 2026 09:01:51 -0800 Subject: [PATCH 04/41] fix(aws-startup-advisor): stop granting GPT-5.4/5.5 an in. CRIS id in the rewriter and Design/Generate checklists bc51f0cc scoped the in. prefix to GPT-5.6 in the catalog and schema, but three consumers of that rule still said us./in./global. for GPT-5.4/5.5: agents/llm2bedrock-code-rewriter.md, gcp-to-aws design-ai.md's checklist, and azure-to-aws generate-artifacts-ai.md's checklist. Their model cards publish only us. and global. ids. Each now reads: us./in./global. for GPT-5.6; us./global. only for GPT-6, GPT-5.5 and GPT-5.4. Adds tests/test_gpt54_55_cris_prefix_scope.py, which reads the catalog, the shared/Azure schemas, the guardrails guide, and the three consumer files and fails on any window that pairs GPT-5.4/5.5 with a grouped us./in./global. grant without scoping in. to GPT-5.6. Verified to fail against the previous wording of all three files. --- .../agents/llm2bedrock-code-rewriter.md | 2 +- .../phases/generate/generate-artifacts-ai.md | 4 +- .../references/phases/design/design-ai.md | 3 +- .../tests/test_gpt54_55_cris_prefix_scope.py | 76 +++++++++++++++++++ 4 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 plugins/aws-startup-advisor/tests/test_gpt54_55_cris_prefix_scope.py diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md b/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md index d4065c4a5..38f89db7c 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md @@ -257,7 +257,7 @@ client = anthropic.Anthropic( Preserve compatible SDK request/response shapes. A model version change still requires §9 target checks, including typed response parsing for Opus 5.5. Apply the Chat Completions → Responses reshape only when that API change was selected. Preserve the exact validated runtime profile ID or ARN. After applying the changes above, skip the Converse-specific guidance in the rest of §8; the §9 behavior-delta application still applies normally. -**Converse for a proprietary GPT target is id-form-dependent, not family-dependent.** A **bare** proprietary GPT id (`openai.gpt-5.5`, `openai.gpt-5.4`, `openai.gpt-5.6-*`, `openai.gpt-6-*`) is a `bedrock-mantle` id with no `bedrock-runtime` in-region form — a boto3 `converse()` call against a bare id fails at runtime; if your context pairs a bare id with the Converse path, stop and report the contradiction. GPT-5.6, GPT-6 (Astra, Sol, Luna, 6.1 Sol) **and GPT-5.5 / GPT-5.4** (model cards read 2026-11-06: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`; Responses, Chat Completions and Converse on runtime, no Invoke) all have a `bedrock-runtime` path via CRIS ids (`us.`/`in.`/`global.` prefixed): when the plan says `migration_path: runtime_openai_cris`, a Converse, Responses or Chat Completions rewrite against the CRIS id is valid (base URL `bedrock-runtime.{region}.amazonaws.com/openai/v1` for the OpenAI-compatible APIs; note Guardrails are Converse-only and prompt caching Responses-only on runtime). For a Responses-based source app, the mantle express lane above remains the default and smallest change. +**Converse for a proprietary GPT target is id-form-dependent, not family-dependent.** A **bare** proprietary GPT id (`openai.gpt-5.5`, `openai.gpt-5.4`, `openai.gpt-5.6-*`, `openai.gpt-6-*`) is a `bedrock-mantle` id with no `bedrock-runtime` in-region form — a boto3 `converse()` call against a bare id fails at runtime; if your context pairs a bare id with the Converse path, stop and report the contradiction. GPT-5.6, GPT-6 (Astra, Sol, Luna, 6.1 Sol) **and GPT-5.5 / GPT-5.4** (model cards read 2026-11-06: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`; Responses, Chat Completions and Converse on runtime, no Invoke) all have a `bedrock-runtime` path via CRIS ids (`us.`/`in.`/`global.` prefixed for GPT-5.6; `us.`/`global.` for GPT-6, GPT-5.5 and GPT-5.4 — those cards publish no `in.` profile): when the plan says `migration_path: runtime_openai_cris`, a Converse, Responses or Chat Completions rewrite against the CRIS id is valid (base URL `bedrock-runtime.{region}.amazonaws.com/openai/v1` for the OpenAI-compatible APIs; note Guardrails are Converse-only and prompt caching Responses-only on runtime). For a Responses-based source app, the mantle express lane above remains the default and smallest change. ### Converse rewrite (default) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md index 109a9a003..1f732c5fd 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -157,8 +157,8 @@ is AWS-only and ports from gcp unchanged. `global.` CRIS id. Chat/Responses and runtime artifact selection match the saved design. - [ ] No **bare** proprietary GPT model ID (including bare `openai.gpt-5.5` / `openai.gpt-5.4`) paired with a Converse/`bedrock-runtime` path. GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 MAY use - `migration_path: "runtime_openai_cris"` on `bedrock-runtime` with a `us.`/`in.`/`global.` - CRIS id — this is NOT a violation. + `migration_path: "runtime_openai_cris"` on `bedrock-runtime` with a CRIS id (`us.`/`in.`/`global.` + for GPT-5.6; `us.`/`global.` only for GPT-6, GPT-5.5 and GPT-5.4) — this is NOT a violation. - [ ] Plugin attribution strings say "Azure-to-AWS". - [ ] `generation-ai.json` validates; `rollback_plan.mechanism == "feature_flag"`. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index a744a6faf..281804c92 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -518,7 +518,8 @@ Write `aws-design-ai.json` to `$MIGRATION_DIR/`. - [ ] No `bedrock_models[]` entry pairs a **bare** proprietary GPT id (including bare `openai.gpt-5.5` / `openai.gpt-5.4`) with a Converse/`bedrock-runtime` migration path. GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 MAY use `migration_path: "runtime_openai_cris"` on - `bedrock-runtime` with a `us.`/`in.`/`global.` CRIS id — this is NOT a violation. + `bedrock-runtime` with a CRIS id (`us.`/`in.`/`global.` for GPT-5.6; `us.`/`global.` only for + GPT-6, GPT-5.5 and GPT-5.4) — this is NOT a violation. - [ ] `regional_warnings` is present (empty array `[]` if no issues; populated if any service unavailable in target region) - [ ] `multi_model_warnings` is present (empty array `[]` if single model or no coordination issues) - [ ] If `agentic_profile.is_agentic == true`: `agentic_design` object is present with `migration_approach` matching `preferences.json` diff --git a/plugins/aws-startup-advisor/tests/test_gpt54_55_cris_prefix_scope.py b/plugins/aws-startup-advisor/tests/test_gpt54_55_cris_prefix_scope.py new file mode 100644 index 000000000..7db7f9558 --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_gpt54_55_cris_prefix_scope.py @@ -0,0 +1,76 @@ +"""Guard: the `in.` CRIS prefix is a GPT-5.6 fact only. +The GPT-5.4 and GPT-5.5 model cards (read 2026-11-07) publish only `us.` and +`global.` inference-profile ids: + https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html + https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html +The catalog records that; this test makes sure no prose consumer of the catalog +(design/generate checklists, the code-rewriter agent, the shared schema) hands +those two models a grouped `us.`/`in.`/`global.` grant that would let an agent +record `in.openai.gpt-5.4` / `in.openai.gpt-5.5` as a runtime id. +""" +from __future__ import annotations +import json +import re +from pathlib import Path +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +CATALOG = PLUGIN_ROOT / "skills" / "shared" / "ai" / "models" / "openai-bedrock.json" +# Every file that names GPT-5.4 / GPT-5.5 alongside a CRIS-prefix grant. +CONSUMERS = [ + PLUGIN_ROOT / "agents" / "llm2bedrock-code-rewriter.md", + PLUGIN_ROOT / "skills" / "gcp-to-aws" / "references" / "phases" / "design" / "design-ai.md", + PLUGIN_ROOT / "skills" / "azure-to-aws" / "references" / "phases" / "generate" / "generate-artifacts-ai.md", + PLUGIN_ROOT / "skills" / "shared" / "ai" / "schema-design-aws-ai.md", + PLUGIN_ROOT / "skills" / "azure-to-aws" / "references" / "shared" / "schema-design-aws-ai.md", + PLUGIN_ROOT / "skills" / "shared" / "ai" / "ai-migration-guardrails.md", +] +_GROUPED_PREFIX = re.compile(r"`?us\.`?\s*/\s*`?in\.`?\s*/\s*`?global\.`?") +_GPT_54_55 = re.compile(r"GPT-5\.[45]|gpt-5\.[45]", re.IGNORECASE) +def _sentences_granting_in_to_gpt54_55(text: str) -> list[str]: + """Return sentence-ish windows that mention GPT-5.4/5.5 AND the grouped + us./in./global. prefix WITHOUT also scoping `in.` to GPT-5.6 in that window.""" + hits: list[str] = [] + for m in _GROUPED_PREFIX.finditer(text): + window = text[max(0, m.start() - 320) : m.end() + 320] + if not _GPT_54_55.search(window): + continue + # Acceptable only when the same window explicitly scopes `in.` to GPT-5.6 + # and gives GPT-5.4/5.5 their own `us.`/`global.` grant. + scoped = re.search(r"GPT-5\.6", window) and re.search( + r"`?us\.`?\s*/\s*`?global\.`?\s*(only|for|;)", window + ) + if not scoped: + hits.append(" ".join(window.split())) + return hits +def test_catalog_inference_profiles_for_gpt54_55_are_us_and_global_only() -> None: + catalog = json.loads(CATALOG.read_text()) + for key in ("openai_gpt_5_4", "openai_gpt_5_5"): + assert set(catalog["models"][key]["inference_profiles"]) == {"us", "global"}, key +def test_no_prose_consumer_grants_in_prefix_to_gpt54_55() -> None: + offenders: dict[str, list[str]] = {} + for path in CONSUMERS: + assert path.exists(), path + hits = _sentences_granting_in_to_gpt54_55(path.read_text()) + if hits: + offenders[str(path.relative_to(PLUGIN_ROOT))] = hits + assert not offenders, ( + "GPT-5.4 / GPT-5.5 are granted a us./in./global. CRIS id; their cards publish " + "us. and global. only. Scope `in.` to GPT-5.6:\n" + + "\n".join(f" {k}: {v[0][:200]}" for k, v in offenders.items()) + ) +def test_vendored_copies_match_canonical_for_edited_files() -> None: + pairs = [ + ( + PLUGIN_ROOT / "skills" / "shared" / "ai" / "schema-design-aws-ai.md", + PLUGIN_ROOT / "skills" / "gcp-to-aws" / "references" / "vendored" / "ai" / "schema-design-aws-ai.md", + ), + ( + PLUGIN_ROOT / "skills" / "shared" / "ai" / "schema-design-aws-ai.md", + PLUGIN_ROOT / "skills" / "azure-to-aws" / "references" / "vendored" / "ai" / "schema-design-aws-ai.md", + ), + ( + CATALOG, + PLUGIN_ROOT / "skills" / "agent-advisor" / "references" / "vendored" / "ai" / "models" / "openai-bedrock.json", + ), + ] + for canonical, vendored in pairs: + assert canonical.read_bytes() == vendored.read_bytes(), f"{vendored} drifted from {canonical}" From 6b873f8c9c1752f891c5a3195ce15bacf0cea839 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 7 Nov 2026 09:14:47 -0800 Subject: [PATCH 05/41] fix(aws-startup-advisor): grant the in. CRIS prefix to GPT-5.6 Terra and Luna only, not Sol The GPT-5.6 Sol card (read 2026-11-07) publishes us.openai.gpt-5.6-sol and global.openai.gpt-5.6-sol only; Terra and Luna publish us./in./us-gov./global. Prior wording granted in. to "GPT-5.6" family-wide in the catalog strength, the rewriter, the GCP Design and Azure Generate checklists, both schema copies, and ai-migration-guardrails.md ("GPT-5.6 Sol/Terra/Luna (us./in./global.)"). Each now reads: us./in./global. for GPT-5.6 Terra and Luna; us./global. only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4. tests/test_gpt54_55_cris_prefix_scope.py now also scans the catalog strength text and rejects any grouped in. grant to GPT-5.6 that does not name Terra/Luna and place Sol in the us./global.-only set. Verified to fail against the dfd07bdc wording of every file above. --- .../agents/llm2bedrock-code-rewriter.md | 2 +- .../vendored/ai/models/openai-bedrock.json | 2 +- .../test_openai_model_recommendation.py | 4 +- .../phases/generate/generate-artifacts-ai.md | 3 +- .../references/shared/schema-design-aws-ai.md | 8 +-- .../vendored/ai/ai-migration-guardrails.md | 2 +- .../vendored/ai/schema-design-aws-ai.md | 8 +-- .../references/phases/design/design-ai.md | 4 +- .../vendored/ai/ai-migration-guardrails.md | 2 +- .../vendored/ai/schema-design-aws-ai.md | 8 +-- .../shared/ai/ai-migration-guardrails.md | 2 +- .../shared/ai/models/openai-bedrock.json | 2 +- .../skills/shared/ai/schema-design-aws-ai.md | 8 +-- .../tests/test_gpt54_55_cris_prefix_scope.py | 65 ++++++++++++------- 14 files changed, 72 insertions(+), 48 deletions(-) diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md b/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md index 38f89db7c..d46296670 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md @@ -257,7 +257,7 @@ client = anthropic.Anthropic( Preserve compatible SDK request/response shapes. A model version change still requires §9 target checks, including typed response parsing for Opus 5.5. Apply the Chat Completions → Responses reshape only when that API change was selected. Preserve the exact validated runtime profile ID or ARN. After applying the changes above, skip the Converse-specific guidance in the rest of §8; the §9 behavior-delta application still applies normally. -**Converse for a proprietary GPT target is id-form-dependent, not family-dependent.** A **bare** proprietary GPT id (`openai.gpt-5.5`, `openai.gpt-5.4`, `openai.gpt-5.6-*`, `openai.gpt-6-*`) is a `bedrock-mantle` id with no `bedrock-runtime` in-region form — a boto3 `converse()` call against a bare id fails at runtime; if your context pairs a bare id with the Converse path, stop and report the contradiction. GPT-5.6, GPT-6 (Astra, Sol, Luna, 6.1 Sol) **and GPT-5.5 / GPT-5.4** (model cards read 2026-11-06: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`; Responses, Chat Completions and Converse on runtime, no Invoke) all have a `bedrock-runtime` path via CRIS ids (`us.`/`in.`/`global.` prefixed for GPT-5.6; `us.`/`global.` for GPT-6, GPT-5.5 and GPT-5.4 — those cards publish no `in.` profile): when the plan says `migration_path: runtime_openai_cris`, a Converse, Responses or Chat Completions rewrite against the CRIS id is valid (base URL `bedrock-runtime.{region}.amazonaws.com/openai/v1` for the OpenAI-compatible APIs; note Guardrails are Converse-only and prompt caching Responses-only on runtime). For a Responses-based source app, the mantle express lane above remains the default and smallest change. +**Converse for a proprietary GPT target is id-form-dependent, not family-dependent.** A **bare** proprietary GPT id (`openai.gpt-5.5`, `openai.gpt-5.4`, `openai.gpt-5.6-*`, `openai.gpt-6-*`) is a `bedrock-mantle` id with no `bedrock-runtime` in-region form — a boto3 `converse()` call against a bare id fails at runtime; if your context pairs a bare id with the Converse path, stop and report the contradiction. GPT-5.6, GPT-6 (Astra, Sol, Luna, 6.1 Sol) **and GPT-5.5 / GPT-5.4** (model cards read 2026-11-06: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`; Responses, Chat Completions and Converse on runtime, no Invoke) all have a `bedrock-runtime` path via CRIS ids (`us.`/`in.`/`global.` prefixed for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4 — those cards publish no `in.` profile): when the plan says `migration_path: runtime_openai_cris`, a Converse, Responses or Chat Completions rewrite against the CRIS id is valid (base URL `bedrock-runtime.{region}.amazonaws.com/openai/v1` for the OpenAI-compatible APIs; note Guardrails are Converse-only and prompt caching Responses-only on runtime). For a Responses-based source app, the mantle express lane above remains the default and smallest change. ### Converse rewrite (default) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json index 48758f3c9..cf642646e 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json @@ -51,7 +51,7 @@ "strengths": [ "Bedrock-native multi-model request surface", "Supports Bedrock Guardrails (Converse API only), invocation logging, and CloudWatch", - "For GPT-5.6 this is a SAME-MODEL governance path via CRIS (us./in./global. ids). For GPT-5.5 and GPT-5.4 the runtime ids are us. and global. only (their model cards publish no in. profile). The base URL for the OpenAI-compatible APIs is bedrock-runtime.{region}.amazonaws.com/openai/v1. The model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" + "This is a SAME-MODEL governance path via CRIS. GPT-5.6 Terra and Luna publish us./in./global. ids; us./global. only for GPT-5.6 Sol, GPT-5.5 and GPT-5.4 (their model cards list no in. profile; cards read 2026-11-07). The base URL for the OpenAI-compatible APIs is bedrock-runtime.{region}.amazonaws.com/openai/v1. The model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" ], "constraints": [ "GPT-5.5 / GPT-5.4 on runtime are CRIS-only (us. and global. ids; no in-region form) per their model cards (read 2026-11-06); the cards publish In-Region (1.10x) pricing only, so do not assume Global CRIS price parity for them", diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py index c506f6830..33c5b9d9e 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py @@ -127,7 +127,9 @@ def test_runtime_converse_path_text_does_not_grant_in_profile_to_gpt54_gpt55(): strengths = "\n".join(path["strengths"]) constraints = "\n".join(path["constraints"]) assert "For GPT-5.6, GPT-5.5 and GPT-5.4" not in strengths - assert "GPT-5.5 and GPT-5.4 the runtime ids are us. and global. only" in strengths + # in. exists for GPT-5.6 Terra/Luna only; Sol, 5.5 and 5.4 are us./global. + assert "GPT-5.6 Terra and Luna publish us./in./global. ids" in strengths + assert "us./global. only for GPT-5.6 Sol, GPT-5.5 and GPT-5.4" in strengths assert "For GPT-5.6, runtime does not support server-side tool use" in constraints assert "GPT-5.5 / GPT-5.4 on runtime" in constraints assert "structured outputs and implicit prompt caching are Responses-only" in constraints diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md index 1f732c5fd..554f2b830 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -158,7 +158,8 @@ is AWS-only and ports from gcp unchanged. - [ ] No **bare** proprietary GPT model ID (including bare `openai.gpt-5.5` / `openai.gpt-5.4`) paired with a Converse/`bedrock-runtime` path. GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 MAY use `migration_path: "runtime_openai_cris"` on `bedrock-runtime` with a CRIS id (`us.`/`in.`/`global.` - for GPT-5.6; `us.`/`global.` only for GPT-6, GPT-5.5 and GPT-5.4) — this is NOT a violation. + for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4) — + this is NOT a violation. - [ ] Plugin attribution strings say "Azure-to-AWS". - [ ] `generation-ai.json` validates; `rollback_plan.mechanism == "feature_flag"`. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md index 9d5c929aa..7788f9b6d 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md @@ -127,8 +127,8 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` lists it there (GPT-6, GPT-5.5, GPT-5.4). `mantle_openai_responses` (Azure OpenAI / OpenAI source, model on Bedrock, region carries it — keep the SDK, `model_change: false`) · `runtime_openai_cris` (GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 -same-model move served via `bedrock-runtime` CRIS — `us.`/`in.`/`global.` prefixed for GPT-6 / -GPT-5.6, `us.`/`global.` only for GPT-5.5 / GPT-5.4 — `model_change: false`) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set +same-model move served via `bedrock-runtime` CRIS — `us.`/`in.`/`global.` prefixed for GPT-5.6 +Terra / Luna, `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4 — `model_change: false`) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, intelligent prompt routing, or another feature the same-model CRIS path itself cannot serve) · `gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK swap). A **bare** @@ -207,8 +207,8 @@ rubric file (`ai.md`, `ai-azure-openai-to-bedrock.md`, or a `vendored/ai/*` ref) - [ ] No `bedrock_models[]` entry pairs a **bare** proprietary GPT id (including bare `openai.gpt-5.5` / `openai.gpt-5.4`) with a `converse`/`bedrock-runtime` migration path. GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 MAY use `migration_path: "runtime_openai_cris"` on - `bedrock-runtime` with a CRIS id (`us.`/`in.`/`global.` for GPT-6 / GPT-5.6, `us.`/`global.` - only for GPT-5.5 / GPT-5.4) — this is NOT a violation. + `bedrock-runtime` with a CRIS id (`us.`/`in.`/`global.` for GPT-5.6 Terra / Luna, `us.`/`global.` + only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4) — this is NOT a violation. - [ ] If `ai_source` is `azure_openai` or `openai`: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-migration-guardrails.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-migration-guardrails.md index 1ad1b0a09..d457a12a9 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-migration-guardrails.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-migration-guardrails.md @@ -48,7 +48,7 @@ Inference on `bedrock-mantle` for these models is governed by **two per-model, p | Medium | Medium | Monitor 429s against **token** throughput, not request rate; enable prompt caching | | High | High | Enable prompt caching first (cached input is exempt from input TPM), then request a quota increase | -**Every proprietary GPT model on Bedrock has a `bedrock-runtime` path via CRIS inference profiles.** GPT-5.6 Sol/Terra/Luna (`us.`/`in.`/`global.`), GPT-6 Astra/Sol/Luna/6.1 Sol/6.1 Terra (`us.`/`global.`), and GPT-5.5 / GPT-5.4 (`us.`/`global.`; per their model cards read 2026-11-06, [GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), [GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)) all list `bedrock-runtime` with Responses, Chat Completions and Converse; none has a runtime in-region form. The model cards recommend runtime for new applications. It is a legitimate endpoint option with its own quota family, and for GPT-5.6 / GPT-6 on Global CRIS it is also the cost-parity option (the GPT-5.5 / GPT-5.4 cards publish In-Region pricing only). A same-model endpoint switch is not a model change; only moving to a Bedrock-native model or `gpt-oss` carries the eval cost of a model change. Scaling levers on the mantle path, in order: +**Every proprietary GPT model on Bedrock has a `bedrock-runtime` path via CRIS inference profiles.** GPT-5.6 Terra and Luna (`us.`/`in.`/`global.`), and `us.`/`global.` only for GPT-5.6 Sol, GPT-6 Astra/Sol/Luna/6.1 Sol/6.1 Terra, and GPT-5.5 / GPT-5.4 (the GPT-5.6 Sol card lists no `in.` profile; per their model cards read 2026-11-06, [GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), [GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)) all list `bedrock-runtime` with Responses, Chat Completions and Converse; none has a runtime in-region form. The model cards recommend runtime for new applications. It is a legitimate endpoint option with its own quota family, and for GPT-5.6 / GPT-6 on Global CRIS it is also the cost-parity option (the GPT-5.5 / GPT-5.4 cards publish In-Region pricing only). A same-model endpoint switch is not a model change; only moving to a Bedrock-native model or `gpt-oss` carries the eval cost of a model change. Scaling levers on the mantle path, in order: 1. Prompt caching (GPT-5.6 and GPT-6 on mantle; GPT-5.5 / GPT-5.4 list implicit caching on `bedrock-runtime` Responses only) — 90% off cached input and exempt from the input-TPM quota 2. Exponential backoff with a bounded retry count (`max_retries` on the OpenAI SDK) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md index c31359bbc..c82a8eebc 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -193,8 +193,8 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `mantle_openai_chat` (same, preserving a Chat Completions source on a model whose card lists Chat Completions on mantle — GPT-6, GPT-5.5, GPT-5.4) · `runtime_openai_cris` (same-model move served via -`bedrock-runtime` with a CRIS id, `model_change: false`; `us.`/`in.`/`global.` for GPT-6 and -GPT-5.6, `us.`/`global.` only for GPT-5.5 and GPT-5.4, per their model cards) · `converse` (Bedrock-native +`bedrock-runtime` with a CRIS id, `model_change: false`; `us.`/`in.`/`global.` for GPT-5.6 Terra +and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4, per their model cards) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, intelligent prompt routing, or another feature no same-model path serves) · `gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK @@ -263,8 +263,8 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. - [ ] No `bedrock_models[]` entry pairs a **bare** proprietary `openai.gpt-*` model ID with a `converse`/`bedrock-runtime` migration path (bare ids are mantle ids; a `bedrock-runtime` - target carries a CRIS id: `us.`/`in.`/`global.` for GPT-6 / GPT-5.6, `us.`/`global.` only for - GPT-5.5 / GPT-5.4). + target carries a CRIS id: `us.`/`in.`/`global.` for GPT-5.6 Terra / Luna; `us.`/`global.` only + for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4). - [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves to direct OpenAI: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index 281804c92..a03f7de63 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -518,8 +518,8 @@ Write `aws-design-ai.json` to `$MIGRATION_DIR/`. - [ ] No `bedrock_models[]` entry pairs a **bare** proprietary GPT id (including bare `openai.gpt-5.5` / `openai.gpt-5.4`) with a Converse/`bedrock-runtime` migration path. GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 MAY use `migration_path: "runtime_openai_cris"` on - `bedrock-runtime` with a CRIS id (`us.`/`in.`/`global.` for GPT-5.6; `us.`/`global.` only for - GPT-6, GPT-5.5 and GPT-5.4) — this is NOT a violation. + `bedrock-runtime` with a CRIS id (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; + `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4) — this is NOT a violation. - [ ] `regional_warnings` is present (empty array `[]` if no issues; populated if any service unavailable in target region) - [ ] `multi_model_warnings` is present (empty array `[]` if single model or no coordination issues) - [ ] If `agentic_profile.is_agentic == true`: `agentic_design` object is present with `migration_approach` matching `preferences.json` diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-migration-guardrails.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-migration-guardrails.md index 1ad1b0a09..d457a12a9 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-migration-guardrails.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-migration-guardrails.md @@ -48,7 +48,7 @@ Inference on `bedrock-mantle` for these models is governed by **two per-model, p | Medium | Medium | Monitor 429s against **token** throughput, not request rate; enable prompt caching | | High | High | Enable prompt caching first (cached input is exempt from input TPM), then request a quota increase | -**Every proprietary GPT model on Bedrock has a `bedrock-runtime` path via CRIS inference profiles.** GPT-5.6 Sol/Terra/Luna (`us.`/`in.`/`global.`), GPT-6 Astra/Sol/Luna/6.1 Sol/6.1 Terra (`us.`/`global.`), and GPT-5.5 / GPT-5.4 (`us.`/`global.`; per their model cards read 2026-11-06, [GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), [GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)) all list `bedrock-runtime` with Responses, Chat Completions and Converse; none has a runtime in-region form. The model cards recommend runtime for new applications. It is a legitimate endpoint option with its own quota family, and for GPT-5.6 / GPT-6 on Global CRIS it is also the cost-parity option (the GPT-5.5 / GPT-5.4 cards publish In-Region pricing only). A same-model endpoint switch is not a model change; only moving to a Bedrock-native model or `gpt-oss` carries the eval cost of a model change. Scaling levers on the mantle path, in order: +**Every proprietary GPT model on Bedrock has a `bedrock-runtime` path via CRIS inference profiles.** GPT-5.6 Terra and Luna (`us.`/`in.`/`global.`), and `us.`/`global.` only for GPT-5.6 Sol, GPT-6 Astra/Sol/Luna/6.1 Sol/6.1 Terra, and GPT-5.5 / GPT-5.4 (the GPT-5.6 Sol card lists no `in.` profile; per their model cards read 2026-11-06, [GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), [GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)) all list `bedrock-runtime` with Responses, Chat Completions and Converse; none has a runtime in-region form. The model cards recommend runtime for new applications. It is a legitimate endpoint option with its own quota family, and for GPT-5.6 / GPT-6 on Global CRIS it is also the cost-parity option (the GPT-5.5 / GPT-5.4 cards publish In-Region pricing only). A same-model endpoint switch is not a model change; only moving to a Bedrock-native model or `gpt-oss` carries the eval cost of a model change. Scaling levers on the mantle path, in order: 1. Prompt caching (GPT-5.6 and GPT-6 on mantle; GPT-5.5 / GPT-5.4 list implicit caching on `bedrock-runtime` Responses only) — 90% off cached input and exempt from the input-TPM quota 2. Exponential backoff with a bounded retry count (`max_retries` on the OpenAI SDK) diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md index c31359bbc..c82a8eebc 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -193,8 +193,8 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `mantle_openai_chat` (same, preserving a Chat Completions source on a model whose card lists Chat Completions on mantle — GPT-6, GPT-5.5, GPT-5.4) · `runtime_openai_cris` (same-model move served via -`bedrock-runtime` with a CRIS id, `model_change: false`; `us.`/`in.`/`global.` for GPT-6 and -GPT-5.6, `us.`/`global.` only for GPT-5.5 and GPT-5.4, per their model cards) · `converse` (Bedrock-native +`bedrock-runtime` with a CRIS id, `model_change: false`; `us.`/`in.`/`global.` for GPT-5.6 Terra +and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4, per their model cards) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, intelligent prompt routing, or another feature no same-model path serves) · `gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK @@ -263,8 +263,8 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. - [ ] No `bedrock_models[]` entry pairs a **bare** proprietary `openai.gpt-*` model ID with a `converse`/`bedrock-runtime` migration path (bare ids are mantle ids; a `bedrock-runtime` - target carries a CRIS id: `us.`/`in.`/`global.` for GPT-6 / GPT-5.6, `us.`/`global.` only for - GPT-5.5 / GPT-5.4). + target carries a CRIS id: `us.`/`in.`/`global.` for GPT-5.6 Terra / Luna; `us.`/`global.` only + for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4). - [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves to direct OpenAI: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-migration-guardrails.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-migration-guardrails.md index 1ad1b0a09..d457a12a9 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-migration-guardrails.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-migration-guardrails.md @@ -48,7 +48,7 @@ Inference on `bedrock-mantle` for these models is governed by **two per-model, p | Medium | Medium | Monitor 429s against **token** throughput, not request rate; enable prompt caching | | High | High | Enable prompt caching first (cached input is exempt from input TPM), then request a quota increase | -**Every proprietary GPT model on Bedrock has a `bedrock-runtime` path via CRIS inference profiles.** GPT-5.6 Sol/Terra/Luna (`us.`/`in.`/`global.`), GPT-6 Astra/Sol/Luna/6.1 Sol/6.1 Terra (`us.`/`global.`), and GPT-5.5 / GPT-5.4 (`us.`/`global.`; per their model cards read 2026-11-06, [GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), [GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)) all list `bedrock-runtime` with Responses, Chat Completions and Converse; none has a runtime in-region form. The model cards recommend runtime for new applications. It is a legitimate endpoint option with its own quota family, and for GPT-5.6 / GPT-6 on Global CRIS it is also the cost-parity option (the GPT-5.5 / GPT-5.4 cards publish In-Region pricing only). A same-model endpoint switch is not a model change; only moving to a Bedrock-native model or `gpt-oss` carries the eval cost of a model change. Scaling levers on the mantle path, in order: +**Every proprietary GPT model on Bedrock has a `bedrock-runtime` path via CRIS inference profiles.** GPT-5.6 Terra and Luna (`us.`/`in.`/`global.`), and `us.`/`global.` only for GPT-5.6 Sol, GPT-6 Astra/Sol/Luna/6.1 Sol/6.1 Terra, and GPT-5.5 / GPT-5.4 (the GPT-5.6 Sol card lists no `in.` profile; per their model cards read 2026-11-06, [GPT-5.4](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html), [GPT-5.5](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)) all list `bedrock-runtime` with Responses, Chat Completions and Converse; none has a runtime in-region form. The model cards recommend runtime for new applications. It is a legitimate endpoint option with its own quota family, and for GPT-5.6 / GPT-6 on Global CRIS it is also the cost-parity option (the GPT-5.5 / GPT-5.4 cards publish In-Region pricing only). A same-model endpoint switch is not a model change; only moving to a Bedrock-native model or `gpt-oss` carries the eval cost of a model change. Scaling levers on the mantle path, in order: 1. Prompt caching (GPT-5.6 and GPT-6 on mantle; GPT-5.5 / GPT-5.4 list implicit caching on `bedrock-runtime` Responses only) — 90% off cached input and exempt from the input-TPM quota 2. Exponential backoff with a bounded retry count (`max_retries` on the OpenAI SDK) diff --git a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json index 48758f3c9..cf642646e 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json @@ -51,7 +51,7 @@ "strengths": [ "Bedrock-native multi-model request surface", "Supports Bedrock Guardrails (Converse API only), invocation logging, and CloudWatch", - "For GPT-5.6 this is a SAME-MODEL governance path via CRIS (us./in./global. ids). For GPT-5.5 and GPT-5.4 the runtime ids are us. and global. only (their model cards publish no in. profile). The base URL for the OpenAI-compatible APIs is bedrock-runtime.{region}.amazonaws.com/openai/v1. The model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" + "This is a SAME-MODEL governance path via CRIS. GPT-5.6 Terra and Luna publish us./in./global. ids; us./global. only for GPT-5.6 Sol, GPT-5.5 and GPT-5.4 (their model cards list no in. profile; cards read 2026-11-07). The base URL for the OpenAI-compatible APIs is bedrock-runtime.{region}.amazonaws.com/openai/v1. The model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" ], "constraints": [ "GPT-5.5 / GPT-5.4 on runtime are CRIS-only (us. and global. ids; no in-region form) per their model cards (read 2026-11-06); the cards publish In-Region (1.10x) pricing only, so do not assume Global CRIS price parity for them", diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md index c31359bbc..c82a8eebc 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md @@ -193,8 +193,8 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `mantle_openai_chat` (same, preserving a Chat Completions source on a model whose card lists Chat Completions on mantle — GPT-6, GPT-5.5, GPT-5.4) · `runtime_openai_cris` (same-model move served via -`bedrock-runtime` with a CRIS id, `model_change: false`; `us.`/`in.`/`global.` for GPT-6 and -GPT-5.6, `us.`/`global.` only for GPT-5.5 and GPT-5.4, per their model cards) · `converse` (Bedrock-native +`bedrock-runtime` with a CRIS id, `model_change: false`; `us.`/`in.`/`global.` for GPT-5.6 Terra +and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4, per their model cards) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, intelligent prompt routing, or another feature no same-model path serves) · `gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK @@ -263,8 +263,8 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` `recommend_stay`, `honest_assessment_reason` names a non-cost blocker. - [ ] No `bedrock_models[]` entry pairs a **bare** proprietary `openai.gpt-*` model ID with a `converse`/`bedrock-runtime` migration path (bare ids are mantle ids; a `bedrock-runtime` - target carries a CRIS id: `us.`/`in.`/`global.` for GPT-6 / GPT-5.6, `us.`/`global.` only for - GPT-5.5 / GPT-5.4). + target carries a CRIS id: `us.`/`in.`/`global.` for GPT-5.6 Terra / Luna; `us.`/`global.` only + for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4). - [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves to direct OpenAI: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. diff --git a/plugins/aws-startup-advisor/tests/test_gpt54_55_cris_prefix_scope.py b/plugins/aws-startup-advisor/tests/test_gpt54_55_cris_prefix_scope.py index 7db7f9558..d1d71ba10 100644 --- a/plugins/aws-startup-advisor/tests/test_gpt54_55_cris_prefix_scope.py +++ b/plugins/aws-startup-advisor/tests/test_gpt54_55_cris_prefix_scope.py @@ -1,12 +1,19 @@ -"""Guard: the `in.` CRIS prefix is a GPT-5.6 fact only. -The GPT-5.4 and GPT-5.5 model cards (read 2026-11-07) publish only `us.` and -`global.` inference-profile ids: +"""Guard: the `in.` CRIS prefix belongs to GPT-5.6 Terra and Luna only. +Public model cards (read 2026-11-07): + GPT-5.6 Terra / Luna us. / in. / us-gov. / global. + GPT-5.6 Sol us. / global. + GPT-5.4, GPT-5.5 us. / global. + https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html + https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html + https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html -The catalog records that; this test makes sure no prose consumer of the catalog -(design/generate checklists, the code-rewriter agent, the shared schema) hands -those two models a grouped `us.`/`in.`/`global.` grant that would let an agent -record `in.openai.gpt-5.4` / `in.openai.gpt-5.5` as a runtime id. +This test makes sure no prose consumer of the catalog (design/generate +checklists, the code-rewriter agent, the shared schema, the catalog strength +text) hands GPT-5.4 / GPT-5.5 a grouped `us.`/`in.`/`global.` grant, and that +any `in.` grant to "GPT-5.6" names Terra/Luna rather than the whole family, +so an agent cannot record `in.openai.gpt-5.6-sol`, `in.openai.gpt-5.4` or +`in.openai.gpt-5.5` as a runtime id. """ from __future__ import annotations import json @@ -25,36 +32,50 @@ ] _GROUPED_PREFIX = re.compile(r"`?us\.`?\s*/\s*`?in\.`?\s*/\s*`?global\.`?") _GPT_54_55 = re.compile(r"GPT-5\.[45]|gpt-5\.[45]", re.IGNORECASE) -def _sentences_granting_in_to_gpt54_55(text: str) -> list[str]: - """Return sentence-ish windows that mention GPT-5.4/5.5 AND the grouped - us./in./global. prefix WITHOUT also scoping `in.` to GPT-5.6 in that window.""" +_TERRA_LUNA_SCOPED = re.compile(r"GPT-5\.6\s+Terra\s*(and|/)\s*(GPT-5\.6\s+)?Luna", re.IGNORECASE) +_SOL_EXCLUDED = re.compile(r"`?us\.`?\s*/\s*`?global\.`?\s*(only\s+)?for[^.;)]*GPT-5\.6\s+Sol", re.IGNORECASE) +_US_GLOBAL_GRANT = re.compile(r"`?us\.`?\s*/\s*`?global\.`?\s*(only|for|;)") +def _bad_in_grants(text: str) -> list[str]: + """Return windows around a grouped us./in./global. grant that either + (a) cover GPT-5.4/5.5 without giving them their own us./global. grant, or + (b) grant `in.` to "GPT-5.6" family-wide instead of to Terra/Luna with Sol + explicitly in the us./global.-only set.""" hits: list[str] = [] for m in _GROUPED_PREFIX.finditer(text): - window = text[max(0, m.start() - 320) : m.end() + 320] - if not _GPT_54_55.search(window): + window = text[max(0, m.start() - 360) : m.end() + 360] + mentions_54_55 = bool(_GPT_54_55.search(window)) + mentions_56 = bool(re.search(r"GPT-5\.6", window)) + if not (mentions_54_55 or mentions_56): continue - # Acceptable only when the same window explicitly scopes `in.` to GPT-5.6 - # and gives GPT-5.4/5.5 their own `us.`/`global.` grant. - scoped = re.search(r"GPT-5\.6", window) and re.search( - r"`?us\.`?\s*/\s*`?global\.`?\s*(only|for|;)", window - ) - if not scoped: + ok = True + if mentions_54_55 and not _US_GLOBAL_GRANT.search(window): + ok = False + if mentions_56 and not (_TERRA_LUNA_SCOPED.search(window) and _SOL_EXCLUDED.search(window)): + ok = False + if not ok: hits.append(" ".join(window.split())) return hits def test_catalog_inference_profiles_for_gpt54_55_are_us_and_global_only() -> None: catalog = json.loads(CATALOG.read_text()) for key in ("openai_gpt_5_4", "openai_gpt_5_5"): assert set(catalog["models"][key]["inference_profiles"]) == {"us", "global"}, key -def test_no_prose_consumer_grants_in_prefix_to_gpt54_55() -> None: +def test_no_prose_consumer_grants_in_prefix_to_gpt54_55_or_gpt56_sol() -> None: offenders: dict[str, list[str]] = {} + # The catalog's runtime_converse path strength is prose an agent reads too. + catalog = json.loads(CATALOG.read_text()) + strength_text = "\n".join(catalog["paths"]["runtime_converse"]["strengths"]) + hits = _bad_in_grants(strength_text) + if hits: + offenders["skills/shared/ai/models/openai-bedrock.json (runtime_converse.strengths)"] = hits for path in CONSUMERS: assert path.exists(), path - hits = _sentences_granting_in_to_gpt54_55(path.read_text()) + hits = _bad_in_grants(path.read_text()) if hits: offenders[str(path.relative_to(PLUGIN_ROOT))] = hits assert not offenders, ( - "GPT-5.4 / GPT-5.5 are granted a us./in./global. CRIS id; their cards publish " - "us. and global. only. Scope `in.` to GPT-5.6:\n" + "A grouped us./in./global. CRIS grant covers GPT-5.4 / GPT-5.5 or the whole GPT-5.6 " + "family. Cards (2026-11-07): in. exists for GPT-5.6 Terra/Luna only; GPT-5.6 Sol, " + "GPT-5.4 and GPT-5.5 publish us./global. only:\n" + "\n".join(f" {k}: {v[0][:200]}" for k, v in offenders.items()) ) def test_vendored_copies_match_canonical_for_edited_files() -> None: From 8c37ef0f1e9a83d123d6662637fc6fadb6f068f8 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 7 Nov 2026 18:26:40 -0800 Subject: [PATCH 06/41] =?UTF-8?q?fix(aws-startup-advisor):=20address=20PR?= =?UTF-8?q?=20583=20review=20=E2=80=94=20GPT-5.4/5.5=20default-project=20I?= =?UTF-8?q?AM=20grant,=20Converse=20for=20application=20profiles,=20In-Reg?= =?UTF-8?q?ion-only=20pricing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three inline findings by leon1418 on 6b873f8c, each verified against the public GPT-5.4 / GPT-5.5 model cards (read 2026-11-07): https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html 1. IAM default-project grant (iam_policy.py:29). The cards require bedrock:InvokeModel on arn:aws:bedrock:{region}:{account-id}:project/default in addition to the inference profile. iam_policy.py gated that grant on is_gpt6_cris_model(). Added is_gpt54_55_cris_model() (exact us./global. openai.gpt-5.4 / -5.5 short ids and their inference-profile ARNs; bare mantle ids, GPT-5.6 CRIS ids and unrelated models do not match) and requires_default_project_grant(); generate_policy() now emits the third ARN for those ids. Updated the GPT-6-only sentence in gcp generate-ai.md, gcp/azure generate-artifacts-ai.md Step 3, and the report generator §6.4. 2. Application inference profiles are Converse-only (gcp design-ai.md:425). GCP and Azure Design now require required_client_api: "converse" when application inference profiles are the driving need on GPT-5.5 / GPT-5.4 (and any model whose card lists them Converse-only), forbid leaving the field absent, keep the responses default for a region-only move, and record a conflicting Responses-only need (structured outputs, implicit caching) as an explicit decision instead of a silent drop. Added checklist items to the GCP Validation Checklist and the Azure shared schema (required_client_api section and checklist). 3. Unpublished Geo/Global prices (ai-openai-to-bedrock.md:46). The cards publish commercial In-Region prices only. Scoped the verified ~10% premium to the In-Region option in the canonical guide (Tier 0 rows included), both openai-on-bedrock.md fact bases, the report generator and the pricing-cache rows; a CRIS plan needs a separately sourced Geo/Global price or reports the comparison unavailable, and an In-Region proxy is an unverified planning assumption with no verified premium or savings percentage. bedrock_pricing.lookup() already returns available:false for the CRIS ids; pinned with a test (short ids and profile ARNs, no PriceList call). Tests: test_iam_policy.py TestGpt54Gpt55CrisProjectArn (short ids, profile ARNs, bare/GPT-5.6/Anthropic controls, mixed run); test_bedrock_pricing.py test_gpt54_55_cris_ids_are_unpriced_not_inferred_from_in_region; new tests/test_gpt54_55_runtime_review_round3.py text contracts for all three findings. tools/sync-vendored.py re-run (5 copies synced). Validation (this commit, in the worktree): - mise -C plugins/aws-startup-advisor run ci: exit 0 (test 1407 passed + 222 subtests; test:telemetry 311 passed; heroku-source-review 44/44; lint:md 0 errors; vendored-parity PASS 62 files; skill-refs PASS 1482) - llm-to-bedrock: uv run --project scripts python -m pytest: 453 passed (bare mise python3 shows 10 ModuleNotFoundError: httpx in test_astra_execution_contract.py — environment, not the change) - agent-advisor: python3 -m pytest skills/agent-advisor: 570 passed - tests/test_gpt54_55_cris_prefix_scope.py: still green - python3 tools/validate.py: All validations passed (exit 0) - tools/sync-vendored.py --check: PASS - mise run security (gitleaks): 1 pre-existing upstream false positive at f8d09fd1 (tests/test_bedrock_migration_assessment_discover.py, generic-api-key), unrelated to this PR; nothing new flagged. --- .../agents/llm2bedrock-report-generator.md | 4 +- .../vendored/ai/bedrock-pricing-cache.md | 4 +- .../references/phases/design/design-ai.md | 18 +- .../phases/generate/generate-artifacts-ai.md | 10 +- .../references/shared/openai-on-bedrock.md | 15 +- .../references/shared/schema-design-aws-ai.md | 13 +- .../vendored/ai/ai-openai-to-bedrock.md | 15 +- .../vendored/ai/bedrock-pricing-cache.md | 4 +- .../references/phases/design/design-ai.md | 24 ++- .../references/phases/generate/generate-ai.md | 2 +- .../phases/generate/generate-artifacts-ai.md | 2 +- .../references/shared/openai-on-bedrock.md | 15 +- .../vendored/ai/ai-openai-to-bedrock.md | 15 +- .../vendored/ai/bedrock-pricing-cache.md | 4 +- .../llm-to-bedrock/scripts/iam_policy.py | 39 ++++- .../scripts/test_bedrock_pricing.py | 23 +++ .../llm-to-bedrock/scripts/test_iam_policy.py | 90 ++++++++++ .../skills/shared/ai/ai-openai-to-bedrock.md | 15 +- .../skills/shared/ai/bedrock-pricing-cache.md | 4 +- .../test_gpt54_55_runtime_review_round3.py | 164 ++++++++++++++++++ 20 files changed, 435 insertions(+), 45 deletions(-) create mode 100644 plugins/aws-startup-advisor/tests/test_gpt54_55_runtime_review_round3.py diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md index 9b9de321e..9dd8b8107 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md @@ -143,7 +143,7 @@ The pricing script only covers Bedrock models. For the source provider (OpenAI, **Astra:** use its separately dated Bedrock rates and context/inference tier. Its source price is absent from the table below; verify that price or mark the source comparison unavailable. Do not estimate Astra from a lower GPT tier or assume GPT-5.x parity/premium. -**GPT-5.x same-model pair cost depends on the inference option — state it conditionally, never as a flat figure.** In-Region and Geo CRIS are priced at OpenAI's _data residency_ tier (1.10x standard), so those moves cost ~10% more — quote the increase plainly and carry the non-cost justification (commitments, governance, residency, prompt caching, no behaviour delta). **GPT-5.6 on Global CRIS is priced at the standard rate — cost parity** — applicable only when the workload has no data-residency constraint; when the plan's `migration_path` is `runtime_openai_cris` with a `global.` id, report parity, not a premium. GPT-5.5/5.4 do have `us.`/`global.` CRIS ids on `bedrock-runtime`, but their model cards (read 2026-11-06) publish In-Region pricing only — so the ~10% figure applies to them on every inference option until a Global rate is published; never report parity for them. **If a pair's workload exceeds 272K context**, price the Bedrock side at the long-context tier and show that line separately: GPT-5.6 is 2.0x input / 1.5x output per option; GPT-5.5 is 11.00 / 1.10 cache read / 49.50 and GPT-5.4 is 5.50 / 0.55 cache read / 24.75 per 1M tokens (In-Region, from their cards), applied to all tokens of the request, within a 1,050,000-token context window. +**GPT-5.x same-model pair cost depends on the inference option — state it conditionally, never as a flat figure.** In-Region and Geo CRIS are priced at OpenAI's _data residency_ tier (1.10x standard), so those moves cost ~10% more — quote the increase plainly and carry the non-cost justification (commitments, governance, residency, prompt caching, no behaviour delta). **GPT-5.6 on Global CRIS is priced at the standard rate — cost parity** — applicable only when the workload has no data-residency constraint; when the plan's `migration_path` is `runtime_openai_cris` with a `global.` id, report parity, not a premium. GPT-5.5/5.4 do have `us.`/`global.` CRIS ids on `bedrock-runtime`, but their model cards (read 2026-11-07) publish **commercial In-Region prices only** ("Commercial In-Region prices include a 10% fee over OpenAI rates") — the ~10% premium is verified for their In-Region (mantle) option alone; nothing establishes a Geo or Global CRIS price. For a `runtime_openai_cris` plan on GPT-5.5/5.4, `bedrock_pricing.lookup()` returns `available: false` for the CRIS id and that is correct: do not substitute the In-Region row as a verified rate. Either source the Geo/Global price separately (URL + read date) and add it to `STATIC_FALLBACK`, or take the §6.3 fallback (`Cost comparison unavailable`, `cost_savings_percent: 0`). If you still show an In-Region proxy, label it an **unverified planning assumption** and report no verified premium or savings percentage from it. Never report parity for them. **If a pair's workload exceeds 272K context**, price the Bedrock side at the long-context tier and show that line separately: GPT-5.6 is 2.0x input / 1.5x output per option; GPT-5.5 is 11.00 / 1.10 cache read / 49.50 and GPT-5.4 is 5.50 / 0.55 cache read / 24.75 per 1M tokens (In-Region, from their cards), applied to all tokens of the request, within a 1,050,000-token context window. If the source model is not in this table (likely for any model released after the table's last update — check the provider's public pricing page if you know current rates), note it as a gap in the Risk Assessment section, label the figure "(estimated from ``)", and estimate using the closest listed model's pricing. @@ -571,7 +571,7 @@ uv run --project python /iam_policy.py \ --output /.saws-migrate/iam-policy.json ``` -The script handles the dual-ARN pattern: foundation-model ARNs for plain model IDs and inference-profile ARNs for geo-prefixed IDs (e.g. `us.anthropic.claude-sonnet-4-6`). The output is a ready-to-use IAM policy JSON file. +The script handles the dual-ARN pattern: foundation-model ARNs for plain model IDs and inference-profile ARNs for geo-prefixed IDs (e.g. `us.anthropic.claude-sonnet-4-6`). For a GPT-6 CRIS id and for the GPT-5.5 / GPT-5.4 CRIS ids (`us.`/`global.openai.gpt-5.5`, `us.`/`global.openai.gpt-5.4`, or their `inference-profile/` ARNs) it adds a third resource, `arn:aws:bedrock:::project/default`, because those model cards (GPT-5.4 / GPT-5.5 read 2026-11-07) require `bedrock:InvokeModel` on the default project in addition to the inference profile; do not strip it from the report's policy excerpt. The output is a ready-to-use IAM policy JSON file. **Mantle targets get different actions.** When any target is a bare proprietary OpenAI GPT model (GPT-5.x or `openai.gpt-6-astra`), the script also emits `bedrock-mantle:CreateInference` / `Get*` / `List*` scoped to `arn:aws:bedrock-mantle:::project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` (AWS does not permit narrowing that one). `bedrock:InvokeModel` alone does **not** authorize these models. Two consequences to carry into the report: diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md index a4206583c..ff4fa58e1 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-pricing-cache.md @@ -133,8 +133,8 @@ See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommen | GPT-5.6 Sol | openai.gpt-5.6-sol | OpenAI | 4.40 | 22.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 8.80/33.00) | | GPT-5.6 Terra | openai.gpt-5.6-terra | OpenAI | 2.20 | 13.20 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 4.40/19.80) | | GPT-5.6 Luna | openai.gpt-5.6-luna | OpenAI | 0.22 | 1.32 | 272K | fast | active (mantle in-region + runtime CRIS; 1M tier 0.44/1.98) | -| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 11.00/49.50; card read 2026-11-06) | -| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 5.50/24.75; card read 2026-11-06) | +| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (commercial In-Region price; runtime us./global. CRIS ids exist but carry no published rate; 1M tier 11.00/49.50; card read 2026-11-07) | +| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (commercial In-Region price; runtime us./global. CRIS ids exist but carry no published rate; 1M tier 5.50/24.75; card read 2026-11-07) | | Gemma 4 31B | google.gemma-4-31b | Google | 0.14 | 0.40 | 256K | mid | active (Mantle only) | | Gemma 4 26B-A4B | google.gemma-4-26b-a4b | Google | 0.13 | 0.40 | 256K | efficient | active (Mantle only) | | Gemma 4 E2B | google.gemma-4-e2b | Google | 0.04 | 0.08 | 128K | budget | active (Mantle only) | diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md index 7b4564837..0314ff6ca 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md @@ -241,7 +241,23 @@ Sol/Luna/6.1 Sol. **When Guardrails is the driving need and the design keeps surface — `converse` for Astra and 6.1 Terra; `converse` / `chat_completions` / `invoke` (never `responses`) for 6.1 Sol and the other GPT-6/5.6 CRIS models except Astra and 6.1 Terra. Do not leave the client on `responses` when Guardrails is required.** `Responses` stays the default for a region-only -CRIS move with no Guardrails need. +CRIS move with no Guardrails or application-inference-profile need. + +**Application inference profiles are Converse-only on GPT-5.5, GPT-5.4, GPT-6.1 Terra and Astra — +the requirement MUST reach `required_client_api`.** The GPT-5.4 / GPT-5.5 model cards (read 2026-11-07) +list "Application inference profiles (Converse API only; not supported with Responses or Chat +Completions APIs)" on `bedrock-runtime`. When application inference profiles (cost attribution by +app/environment) are the driving need and the design keeps `runtime_openai_cris`, record +`bedrock_models[].required_client_api: "converse"` for GPT-5.5 / GPT-5.4 and for any other model +whose card or catalog `runtime_converse` path entry lists the feature as Converse-only. Do not leave +the field absent or on `responses` / `chat_completions`: Generate defaults an absent field to +Responses and omits the application profile for an unsupported model/API pair, so the endpoint +switch alone does not deliver the feature. If the same workload also needs a Responses-only +feature on that model (GPT-5.5 / GPT-5.4: structured outputs, implicit prompt caching — see +`openai-on-bedrock.md` § Access Paths), one client API cannot serve both: record the conflict as an +explicit decision in the plan and the Present Summary — name which feature the plan keeps and +which is dropped, or split the workload into two `bedrock_models[]` rows with their own +`required_client_api` — never silently drop either requirement. **GPT-6.1 Terra API constraints:** Guardrails and application inference profiles require `required_client_api: "converse"`. Converse rejects document blocks and images in tool diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md index 554f2b830..b3f83648e 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -80,10 +80,14 @@ diffs outputs, so the user validates behavior before cutover. ## Step 3: `setup_bedrock.sh` Enables the chosen Bedrock model(s), sets up IAM, and (mantle path) the Bedrock API key. For a -`runtime_openai_cris` target whose CRIS id is a GPT-6 model (Astra, Sol, Luna, 6.1 Sol, 6.1 Terra), the IAM +`runtime_openai_cris` target whose CRIS id is a GPT-6 model (Astra, Sol, Luna, 6.1 Sol, 6.1 Terra) or +GPT-5.5 / GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / +`global.openai.gpt-5.4`, or their `inference-profile/` ARNs), the IAM policy must grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` -in addition to the CRIS inference-profile ARN — both required per the GPT-6 model cards; the -mantle `bedrock-mantle:*` policy is unaffected. +in addition to the CRIS inference-profile ARN — both required per the GPT-6 model cards and the +GPT-5.5 / GPT-5.4 cards (read 2026-11-07); `iam_policy.py` emits the third ARN for exactly these +ids. The mantle `bedrock-mantle:*` policy (bare `openai.gpt-5.5` / `openai.gpt-5.4` ids included) +is unaffected. ## Step 3B: `gateway_config.{yaml,py,json}` (gateway users) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md index 7c9cddaa9..86c2781ab 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md @@ -269,8 +269,13 @@ inference-option dimension:** So the honest cost statement is conditional, not flat: a same-model GPT-5.6 move on Global CRIS is **cost-neutral**; the same move in-region or Geo is **~10% more expensive**. The GPT-5.5 / GPT-5.4 cards publish -In-Region pricing only (no Geo/Global rows as of 2026-11-06), so quote ~10% for them on every inference option. The -same Global-CRIS-is-cost-neutral / in-Region-and-Geo-is-~10%-more pattern applies to the GPT-6 family below. +**commercial In-Region pricing only** (no Geo/Global rows as of 2026-11-07; the cards say "Commercial In-Region +prices include a 10% fee over OpenAI rates"), so for them the ~10% premium is verified for the In-Region (mantle) +option alone — nothing establishes a Geo (`us.`) or Global CRIS price. For a GPT-5.5 / GPT-5.4 CRIS plan, source the +Geo/Global rate separately (the card's inference-option table or the Bedrock pricing page, with URL and read date) +or report the comparison unavailable; an In-Region rate used as a proxy is an **unverified planning assumption**, +labelled as such, and must not be turned into a verified premium or savings percentage. Never report parity for +them. The same Global-CRIS-is-cost-neutral / in-Region-and-Geo-is-~10%-more pattern applies to the GPT-6 family below. Never state either number without stating the inference option it belongs to. ### GPT-6 Astra / Sol / Luna / 6.1 Sol — short context (≤272K) @@ -326,8 +331,10 @@ A workload above 272K context must be priced at the long-context tier. ### GPT-5.5 / GPT-5.4 — In-Region pricing only; short- and long-context tiers (model cards read 2026-11-06) -The cards publish only the In-Region inference option (the `us.` / `global.` runtime CRIS ids exist but carry no -separate price row). Long-context rates apply to all input and output tokens of a request above 272K input tokens. +The cards publish only the commercial In-Region inference option (the `us.` / `global.` runtime CRIS ids exist but +carry no separate price row). A `runtime_openai_cris` plan priced from this table is an unverified planning +assumption, not a verified rate — `bedrock_pricing.lookup()` returns `available: false` for these CRIS ids on +purpose. Long-context rates apply to all input and output tokens of a request above 272K input tokens. | Model | Short context ≤272K (in · cache read · out) | Long context >272K (in · cache read · out) | Notes | | ------- | ------------------------------------------- | ------------------------------------------ | ---------------------------------------- | diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md index 7788f9b6d..a78a20e33 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md @@ -150,7 +150,12 @@ move (no Guardrails or other client-surface need). A Guardrails-requiring worklo `runtime_openai_cris` sets it to the surface that is actually Guardrails-capable for that model: `converse` for GPT-6 Astra and GPT-6.1 Terra, or `converse`/`chat_completions`/`invoke` — never `responses` — for GPT-6.1 Sol and the other -GPT-6/5.6 CRIS models except Astra and GPT-6.1 Terra. Terra also requires `converse` for application inference profiles. `generate-artifacts-ai.md`'s CRIS Step 1M reads this field and +GPT-6/5.6 CRIS models except Astra and GPT-6.1 Terra. Application inference profiles also require +`converse` on every model whose card lists them as Converse-only — GPT-6.1 Terra, Astra, and GPT-5.5 / +GPT-5.4 (cards read 2026-11-07: "Converse API only; not supported with Responses or Chat Completions +APIs") — so a workload needing them on `runtime_openai_cris` sets `converse`, never leaves the field +absent; a same-workload Responses-only need (GPT-5.5 / GPT-5.4 structured outputs, implicit caching) +is an explicit design decision, not a silent drop. `generate-artifacts-ai.md`'s CRIS Step 1M reads this field and emits that surface instead of defaulting to `responses.create`. Keep the existing Astra-no-Invoke migration_path note above — `required_client_api` never sets `invoke` for Astra. @@ -209,6 +214,12 @@ rubric file (`ai.md`, `ai-azure-openai-to-bedrock.md`, or a `vendored/ai/*` ref) GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 MAY use `migration_path: "runtime_openai_cris"` on `bedrock-runtime` with a CRIS id (`us.`/`in.`/`global.` for GPT-5.6 Terra / Luna, `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4) — this is NOT a violation. +- [ ] Every `runtime_openai_cris` entry whose driving need includes application inference profiles, + on a model whose card lists them as Converse-only (GPT-5.5, GPT-5.4, GPT-6.1 Terra, Astra), + records `required_client_api: "converse"` — not absent, `responses` or `chat_completions`. + A same-workload Responses-only need (GPT-5.5 / GPT-5.4 structured outputs or implicit + caching) is recorded as an explicit decision, not silently dropped. A region-only CRIS + move keeps the `responses` default. - [ ] If `ai_source` is `azure_openai` or `openai`: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index 78fb426c0..3cc8e290f 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -43,9 +43,14 @@ Two consequences that invert the previous decision logic: CRIS are priced at OpenAI's _data residency_ tier, exactly 1.10x the standard list price, so those moves cost ~10% more. **GPT-5.6 on Global CRIS is priced at the standard list price — cost parity** — available only when the workload has no data-residency constraint. GPT-5.5 / GPT-5.4 also have `us.`/`global.` CRIS ids on - `bedrock-runtime`, but their model cards (read 2026-11-06) publish In-Region pricing only, so quote the ~10% - figure for them on every inference option until a Global rate is published. See - `shared/openai-on-bedrock.md`. The non-cost case — AWS commitments, IAM/VPC/CloudTrail governance, residency, + `bedrock-runtime`, but their model cards (read 2026-11-07) publish **commercial In-Region prices only** + ("Commercial In-Region prices include a 10% fee over OpenAI rates") — nothing on either card establishes a + Geo or Global CRIS price. So the verified ~10% premium for GPT-5.5 / GPT-5.4 is scoped to the commercial + In-Region (mantle) option. For a GPT-5.5 / GPT-5.4 CRIS plan, either source the Geo/Global rate separately + (the card's inference-option table or [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/), + with URL and read date) or report the comparison unavailable; an In-Region rate used as a proxy is an + **unverified planning assumption**, labelled as such, and yields no verified premium or savings percentage. + Never report parity for them. See `shared/openai-on-bedrock.md`. The non-cost case — AWS commitments, IAM/VPC/CloudTrail governance, residency, prompt caching, one vendor relationship — carries the in-region path; never present that path as free or neutral. For Astra, use its separately dated Bedrock rates and verify the source provider's rate before comparing. @@ -74,8 +79,8 @@ GPT-5.4, the target is **the same model on Bedrock**. | GPT-5.6 Sol | GPT-5.6 Sol | `openai.gpt-5.6-sol` / `us.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | | GPT-5.6 Terra | GPT-5.6 Terra | `openai.gpt-5.6-terra` / `us.` `in.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | | GPT-5.6 Luna | GPT-5.6 Luna | `openai.gpt-5.6-luna` / `us.` `in.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | -| GPT-5.5 | GPT-5.5 | `openai.gpt-5.5` / `us.` `global.` prefixed | `strong_migrate` — same model, ~10% over OpenAI std | -| GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` / `us.` `global.` prefixed | `strong_migrate` — same model, ~10% over OpenAI std | +| GPT-5.5 | GPT-5.5 | `openai.gpt-5.5` / `us.` `global.` prefixed | `strong_migrate` — same model; ~10% over OpenAI std In-Region only (no Geo/Global price published) | +| GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` / `us.` `global.` prefixed | `strong_migrate` — same model; ~10% over OpenAI std In-Region only (no Geo/Global price published) | Then apply the **region gate** below. Record `model_change: false` and the chosen path in `aws-design-ai.json` → `ai_architecture.code_migration`: `migration_path: "mantle_openai_chat"` for diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md index a4206583c..ff4fa58e1 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-pricing-cache.md @@ -133,8 +133,8 @@ See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommen | GPT-5.6 Sol | openai.gpt-5.6-sol | OpenAI | 4.40 | 22.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 8.80/33.00) | | GPT-5.6 Terra | openai.gpt-5.6-terra | OpenAI | 2.20 | 13.20 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 4.40/19.80) | | GPT-5.6 Luna | openai.gpt-5.6-luna | OpenAI | 0.22 | 1.32 | 272K | fast | active (mantle in-region + runtime CRIS; 1M tier 0.44/1.98) | -| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 11.00/49.50; card read 2026-11-06) | -| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 5.50/24.75; card read 2026-11-06) | +| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (commercial In-Region price; runtime us./global. CRIS ids exist but carry no published rate; 1M tier 11.00/49.50; card read 2026-11-07) | +| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (commercial In-Region price; runtime us./global. CRIS ids exist but carry no published rate; 1M tier 5.50/24.75; card read 2026-11-07) | | Gemma 4 31B | google.gemma-4-31b | Google | 0.14 | 0.40 | 256K | mid | active (Mantle only) | | Gemma 4 26B-A4B | google.gemma-4-26b-a4b | Google | 0.13 | 0.40 | 256K | efficient | active (Mantle only) | | Gemma 4 E2B | google.gemma-4-e2b | Google | 0.04 | 0.08 | 128K | budget | active (Mantle only) | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index a03f7de63..3e182e26c 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -436,7 +436,23 @@ must verify against Astra's model card per-feature, not assume parity with Sol/L switch, not model switch), it MUST also record `bedrock_models[].required_client_api` so the generated client uses a Guardrails-capable surface — `converse` for Astra and 6.1 Terra; `converse` / `chat_completions` / `invoke` (never `responses`) for 6.1 Sol and the other GPT-6/5.6 CRIS models except Astra and 6.1 Terra. Do not leave the client on `responses` when Guardrails is required.** `Responses` -stays the default for a region-only CRIS move with no Guardrails need. +stays the default for a region-only CRIS move with no Guardrails or application-inference-profile need. + +**Application inference profiles are Converse-only on GPT-5.5, GPT-5.4, GPT-6.1 Terra and Astra — +the requirement MUST reach `required_client_api`.** The GPT-5.4 / GPT-5.5 model cards (read 2026-11-07) +list "Application inference profiles (Converse API only; not supported with Responses or Chat +Completions APIs)" on `bedrock-runtime`. When application inference profiles (cost attribution by +app/environment) are the driving need and the design keeps `runtime_openai_cris`, record +`bedrock_models[].required_client_api: "converse"` for GPT-5.5 / GPT-5.4 and for any other model +whose card or catalog `runtime_converse` path entry lists the feature as Converse-only. Do not leave +the field absent or on `responses` / `chat_completions`: Generate defaults an absent field to +Responses and omits the application profile for an unsupported model/API pair, so the endpoint +switch alone does not deliver the feature. If the same workload also needs a Responses-only +feature on that model (GPT-5.5 / GPT-5.4: structured outputs, implicit prompt caching — see +`openai-on-bedrock.md` § Access Paths), one client API cannot serve both: record the conflict as an +explicit decision in the Part 5 plan and the Present Summary — name which feature the plan keeps +and which is dropped, or split the workload into two `bedrock_models[]` rows with their own +`required_client_api` — never silently drop either requirement. **GPT-6.1 Terra API constraints:** Guardrails and application inference profiles require `required_client_api: "converse"`. Converse rejects document blocks and images in tool @@ -520,6 +536,12 @@ Write `aws-design-ai.json` to `$MIGRATION_DIR/`. GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 MAY use `migration_path: "runtime_openai_cris"` on `bedrock-runtime` with a CRIS id (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4) — this is NOT a violation. +- [ ] Every `runtime_openai_cris` entry whose driving need includes application inference profiles, + on a model whose card lists them as Converse-only (GPT-5.5, GPT-5.4, GPT-6.1 Terra, Astra), + records `required_client_api: "converse"` — not absent, `responses` or `chat_completions`. + A same-workload Responses-only need (GPT-5.5 / GPT-5.4 structured outputs or implicit + caching) is recorded as an explicit decision in the Part 5 plan, not silently dropped. + A region-only CRIS move keeps the `responses` default. - [ ] `regional_warnings` is present (empty array `[]` if no issues; populated if any service unavailable in target region) - [ ] `multi_model_warnings` is present (empty array `[]` if single model or no coordination issues) - [ ] If `agentic_profile.is_agentic == true`: `agentic_design` object is present with `migration_approach` matching `preferences.json` diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md index 5fbeb17bc..5fa5c23ee 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md @@ -31,7 +31,7 @@ Check `preferences.json` → `ai_constraints.ai_framework` to determine the migr **OpenAI SDK users via Mantle (config path for a Responses source; small reshape for a Chat Completions source)** — `ai_framework` = `direct` AND `ai_source` = `openai` AND (`migration_path` **starts with** `mantle` (`mantle`, `mantle_openai_responses`, `mantle_openai_chat`) OR `migration_path` == `"runtime_openai_cris"`): - `mantle` / `mantle_openai_responses`: point the client at `https://bedrock-mantle.{region}.api.aws/openai/v1` — note the `openai/v1` segment for proprietary GPT models; a bare `/v1` 404s. Swap the credential for a Bedrock API key or the auto-refreshing token provider — **not** the existing OpenAI key. Update the model string to the bare Bedrock model ID (`openai.gpt-*` for a same-model move). Grant the `bedrock-mantle:*` actions; `bedrock:InvokeModel` does not authorize these models. -- `runtime_openai_cris` (GPT-5.6/GPT-6 via `bedrock-runtime` CRIS): point the client at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` instead — **not** the mantle URL. Update the model string to the `us.`/`in.`/`global.`-prefixed CRIS inference-profile ID, not the bare ID. Grant standard `bedrock:InvokeModel*` actions against the CRIS inference-profile ARN — **not** `bedrock-mantle:*`. For a GPT-6 CRIS id, also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` alongside the inference-profile ARN (per the GPT-6 model cards). Check `aws-design-ai.json` → `ai_architecture.bedrock_models[].required_client_api`: if non-Responses, emit that surface (e.g. Converse) instead of defaulting to `responses.create` — absent/`responses` keeps the Responses surface for a region-only move. +- `runtime_openai_cris` (GPT-5.6/GPT-6/GPT-5.5/GPT-5.4 via `bedrock-runtime` CRIS): point the client at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` instead — **not** the mantle URL. Update the model string to the CRIS inference-profile ID the selected model's card lists (`us.`/`global.`; `in.` exists for GPT-5.6 Terra and Luna only), not the bare ID. Grant standard `bedrock:InvokeModel*` actions against the CRIS inference-profile ARN — **not** `bedrock-mantle:*`. For a GPT-6 CRIS id, and for the GPT-5.5 / GPT-5.4 CRIS ids (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`, or their `inference-profile/` ARNs), also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` alongside the inference-profile ARN (per the GPT-6 model cards and the GPT-5.5 / GPT-5.4 cards, read 2026-11-07); `iam_policy.py` emits this third ARN for exactly those ids. Check `aws-design-ai.json` → `ai_architecture.bedrock_models[].required_client_api`: if non-Responses, emit that surface (e.g. Converse) instead of defaulting to `responses.create` — absent/`responses` keeps the Responses surface for a region-only move. - **Config-only if the source already calls `responses.create`.** A Chat Completions source for GPT-6 or GPT-5.6 may keep `chat.completions.create`/`choices[0].message` parsing unchanged — Chat Completions is supported on both mantle and runtime CRIS for these models, so reshape to Responses is optional (only for a Responses-only feature or demonstrated constraint). GPT-5.5/GPT-5.4 follow the same rule — their model cards (read 2026-11-06) list Chat Completions on both mantle and runtime, so the reshape is optional for them too; probe the selected API against the target account before committing, as for any model. - Test in staging, validate responses diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md index c2303a9c2..9ef33ae64 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -209,7 +209,7 @@ Generate `ai-migration/setup_bedrock.sh`. - Dry-run by default (`--execute` flag to run for real) - Step 1 — Request model access: List each model from `aws-design-ai.json` → `bedrock_models[].aws_model_id` and the embedding model -- Step 2 — Create IAM role: Trust policy for the compute platform (Lambda, ECS, or EC2 based on `aws-design.json` if present). Match inference permissions to the selected endpoint. Mantle needs `bedrock-mantle:CreateInference` scoped to `arn:aws:bedrock-mantle:{region}:{account_id}:project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` when using bearer credentials. Runtime needs `bedrock:InvokeModel` / `bedrock:InvokeModelWithResponseStream` against the selected foundation-model ARN and, for CRIS, the exact inference-profile ARN. For GPT-6 CRIS, also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default`. Runtime actions do not authorize Mantle. +- Step 2 — Create IAM role: Trust policy for the compute platform (Lambda, ECS, or EC2 based on `aws-design.json` if present). Match inference permissions to the selected endpoint. Mantle needs `bedrock-mantle:CreateInference` scoped to `arn:aws:bedrock-mantle:{region}:{account_id}:project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` when using bearer credentials. Runtime needs `bedrock:InvokeModel` / `bedrock:InvokeModelWithResponseStream` against the selected foundation-model ARN and, for CRIS, the exact inference-profile ARN. For GPT-6 CRIS, and for the GPT-5.5 / GPT-5.4 CRIS ids (`us.`/`global.openai.gpt-5.5`, `us.`/`global.openai.gpt-5.4`, or their `inference-profile/` ARNs — their cards, read 2026-11-07, require it "in addition to the inference profile"), also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default`. A bare `openai.gpt-5.5` / `openai.gpt-5.4` id is a mantle target and gets no such grant. Runtime actions do not authorize Mantle. - If the runtime client uses `BEDROCK_INFERENCE_PROFILE_ARN` from the generated monitoring output, also authorize that exact `application-inference-profile/…` ARN, retaining the backing model/CRIS resources needed for the selected model and fallback. Resolve it from the same output used by the client; do not grant every application profile. Re-run setup after enabling or changing this override so the effective invocation target and IAM remain aligned. See [inference-profile IAM prerequisites](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-prereq.html). - Compliance carry-through: if `preferences.json` → `design_constraints.compliance` is set and not `none`/`unknown` (full-flow Q2 / AI-only Q1.5), the script and its printed notes MUST reflect the design's Step 0.7 constraints — `AWS_REGION` from the compliance-constrained region (GovCloud for fedramp, EU for gdpr), `eu.` inference-profile model IDs for gdpr, and for hipaa a printed warning that Bedrock invocation logs retain original content (KMS-encrypt + restrict IAM on the log group before enabling) - Step 3 — Print required environment variables: `AWS_REGION`, `AI_PROVIDER=bedrock`, model IDs diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md index 492bced1e..beaeb79b7 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md @@ -274,8 +274,13 @@ inference-option dimension:** So the honest cost statement is conditional, not flat: a same-model GPT-5.6 move on Global CRIS is **cost-neutral**; the same move in-region or Geo is **~10% more expensive**. The GPT-5.5 / GPT-5.4 cards publish -In-Region pricing only (no Geo/Global rows as of 2026-11-06), so quote ~10% for them on every inference option. The -same Global-CRIS-is-cost-neutral / in-Region-and-Geo-is-~10%-more pattern applies to the GPT-6 family below. +**commercial In-Region pricing only** (no Geo/Global rows as of 2026-11-07; the cards say "Commercial In-Region +prices include a 10% fee over OpenAI rates"), so for them the ~10% premium is verified for the In-Region (mantle) +option alone — nothing establishes a Geo (`us.`) or Global CRIS price. For a GPT-5.5 / GPT-5.4 CRIS plan, source the +Geo/Global rate separately (the card's inference-option table or the Bedrock pricing page, with URL and read date) +or report the comparison unavailable; an In-Region rate used as a proxy is an **unverified planning assumption**, +labelled as such, and must not be turned into a verified premium or savings percentage. Never report parity for +them. The same Global-CRIS-is-cost-neutral / in-Region-and-Geo-is-~10%-more pattern applies to the GPT-6 family below. Never state either number without stating the inference option it belongs to. ### GPT-6 Astra / Sol / Luna / 6.1 Sol — short context (≤272K) @@ -331,8 +336,10 @@ A workload above 272K context must be priced at the long-context tier. ### GPT-5.5 / GPT-5.4 — In-Region pricing only; short- and long-context tiers (model cards read 2026-11-06) -The cards publish only the In-Region inference option (the `us.` / `global.` runtime CRIS ids exist but carry no -separate price row). Long-context rates apply to all input and output tokens of a request above 272K input tokens. +The cards publish only the commercial In-Region inference option (the `us.` / `global.` runtime CRIS ids exist but +carry no separate price row). A `runtime_openai_cris` plan priced from this table is an unverified planning +assumption, not a verified rate — `bedrock_pricing.lookup()` returns `available: false` for these CRIS ids on +purpose. Long-context rates apply to all input and output tokens of a request above 272K input tokens. | Model | Short context ≤272K (in · cache read · out) | Long context >272K (in · cache read · out) | Notes | | ------- | ------------------------------------------- | ------------------------------------------ | ---------------------------------------- | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index 78fb426c0..3cc8e290f 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -43,9 +43,14 @@ Two consequences that invert the previous decision logic: CRIS are priced at OpenAI's _data residency_ tier, exactly 1.10x the standard list price, so those moves cost ~10% more. **GPT-5.6 on Global CRIS is priced at the standard list price — cost parity** — available only when the workload has no data-residency constraint. GPT-5.5 / GPT-5.4 also have `us.`/`global.` CRIS ids on - `bedrock-runtime`, but their model cards (read 2026-11-06) publish In-Region pricing only, so quote the ~10% - figure for them on every inference option until a Global rate is published. See - `shared/openai-on-bedrock.md`. The non-cost case — AWS commitments, IAM/VPC/CloudTrail governance, residency, + `bedrock-runtime`, but their model cards (read 2026-11-07) publish **commercial In-Region prices only** + ("Commercial In-Region prices include a 10% fee over OpenAI rates") — nothing on either card establishes a + Geo or Global CRIS price. So the verified ~10% premium for GPT-5.5 / GPT-5.4 is scoped to the commercial + In-Region (mantle) option. For a GPT-5.5 / GPT-5.4 CRIS plan, either source the Geo/Global rate separately + (the card's inference-option table or [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/), + with URL and read date) or report the comparison unavailable; an In-Region rate used as a proxy is an + **unverified planning assumption**, labelled as such, and yields no verified premium or savings percentage. + Never report parity for them. See `shared/openai-on-bedrock.md`. The non-cost case — AWS commitments, IAM/VPC/CloudTrail governance, residency, prompt caching, one vendor relationship — carries the in-region path; never present that path as free or neutral. For Astra, use its separately dated Bedrock rates and verify the source provider's rate before comparing. @@ -74,8 +79,8 @@ GPT-5.4, the target is **the same model on Bedrock**. | GPT-5.6 Sol | GPT-5.6 Sol | `openai.gpt-5.6-sol` / `us.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | | GPT-5.6 Terra | GPT-5.6 Terra | `openai.gpt-5.6-terra` / `us.` `in.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | | GPT-5.6 Luna | GPT-5.6 Luna | `openai.gpt-5.6-luna` / `us.` `in.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | -| GPT-5.5 | GPT-5.5 | `openai.gpt-5.5` / `us.` `global.` prefixed | `strong_migrate` — same model, ~10% over OpenAI std | -| GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` / `us.` `global.` prefixed | `strong_migrate` — same model, ~10% over OpenAI std | +| GPT-5.5 | GPT-5.5 | `openai.gpt-5.5` / `us.` `global.` prefixed | `strong_migrate` — same model; ~10% over OpenAI std In-Region only (no Geo/Global price published) | +| GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` / `us.` `global.` prefixed | `strong_migrate` — same model; ~10% over OpenAI std In-Region only (no Geo/Global price published) | Then apply the **region gate** below. Record `model_change: false` and the chosen path in `aws-design-ai.json` → `ai_architecture.code_migration`: `migration_path: "mantle_openai_chat"` for diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md index a4206583c..ff4fa58e1 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-pricing-cache.md @@ -133,8 +133,8 @@ See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommen | GPT-5.6 Sol | openai.gpt-5.6-sol | OpenAI | 4.40 | 22.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 8.80/33.00) | | GPT-5.6 Terra | openai.gpt-5.6-terra | OpenAI | 2.20 | 13.20 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 4.40/19.80) | | GPT-5.6 Luna | openai.gpt-5.6-luna | OpenAI | 0.22 | 1.32 | 272K | fast | active (mantle in-region + runtime CRIS; 1M tier 0.44/1.98) | -| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 11.00/49.50; card read 2026-11-06) | -| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 5.50/24.75; card read 2026-11-06) | +| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (commercial In-Region price; runtime us./global. CRIS ids exist but carry no published rate; 1M tier 11.00/49.50; card read 2026-11-07) | +| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (commercial In-Region price; runtime us./global. CRIS ids exist but carry no published rate; 1M tier 5.50/24.75; card read 2026-11-07) | | Gemma 4 31B | google.gemma-4-31b | Google | 0.14 | 0.40 | 256K | mid | active (Mantle only) | | Gemma 4 26B-A4B | google.gemma-4-26b-a4b | Google | 0.13 | 0.40 | 256K | efficient | active (Mantle only) | | Gemma 4 E2B | google.gemma-4-e2b | Google | 0.04 | 0.08 | 128K | budget | active (Mantle only) | diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py index 0ceaa4f1e..ce486b3ae 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py @@ -61,8 +61,9 @@ def inference_profile_arn(model_id: str, region: str, account_id: str) -> str: def default_project_arn(region: str, account_id: str) -> str: """ARN for the default Bedrock project, scoped to the account/region. - Required alongside the inference-profile ARN for a GPT-6 CRIS id on - bedrock-runtime: every GPT-6 model card's "Call the model" section grants + Required alongside the inference-profile ARN for a GPT-6 CRIS id and for the + GPT-5.5 / GPT-5.4 CRIS ids on bedrock-runtime: every GPT-6 model card's "Call + the model" section, and the GPT-5.4 / GPT-5.5 cards (read 2026-11-07), grant `bedrock:InvokeModel` on both the inference-profile ARN and this ARN.""" return f"arn:{_partition(region)}:bedrock:{region}:{account_id}:project/default" @@ -77,6 +78,35 @@ def is_gpt6_cris_model(model_id: str) -> bool: return base_id.startswith("openai.gpt-6") +# The exact bedrock-runtime ids the GPT-5.4 / GPT-5.5 model cards list (read +# 2026-11-07: us. and global. only — no in. or in-region form). Both cards say the +# identity "also needs bedrock:InvokeModel on your account's default project +# (arn:aws:bedrock:{region}:{account-id}:project/default) in addition to the +# inference profile". +_GPT54_55_CRIS_IDS = frozenset({ + "us.openai.gpt-5.4", + "global.openai.gpt-5.4", + "us.openai.gpt-5.5", + "global.openai.gpt-5.5", +}) + + +def is_gpt54_55_cris_model(model_id: str) -> bool: + """True for exactly `us.`/`global.openai.gpt-5.4` / `-5.5`, as a short id or as + an `arn:...:inference-profile/` ARN. The bare `openai.gpt-5.4` / `-5.5` ids + (mantle) and every other model, GPT-5.6 CRIS ids included, do NOT match.""" + if not is_inference_profile(model_id): + return False + return model_id.rsplit("/", 1)[-1].lower() in _GPT54_55_CRIS_IDS + + +def requires_default_project_grant(model_id: str) -> bool: + """True when the model card for this runtime CRIS id requires + `bedrock:InvokeModel` on the default project ARN alongside the + inference-profile ARN: every GPT-6 CRIS id, and the GPT-5.5 / GPT-5.4 ones.""" + return is_gpt6_cris_model(model_id) or is_gpt54_55_cris_model(model_id) + + def generate_policy(model_ids: list[str], region: str, account_id: str) -> dict: """Build a scoped IAM policy covering exactly the given model IDs. @@ -102,11 +132,12 @@ def generate_policy(model_ids: list[str], region: str, account_id: str) -> dict: resources.append(inference_profile_arn(mid, region, account_id)) base_id = _GEO_PREFIX.sub("", mid.rsplit("/", 1)[-1]) resources.append(foundation_model_arn(base_id, region)) - # A GPT-6 CRIS id's model card "Call the model" section grants + # A GPT-6 CRIS id's model card "Call the model" section, and the + # GPT-5.5 / GPT-5.4 cards for their us./global. CRIS ids, grant # InvokeModel on the inference-profile ARN AND the default project # ARN — both are required, neither replaces the other. GPT-5.6 CRIS # ids and all other runtime targets keep the 2-ARN shape above. - if is_gpt6_cris_model(mid): + if requires_default_project_grant(mid): resources.append(default_project_arn(region, account_id)) else: resources.append(foundation_model_arn(mid, region)) diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_bedrock_pricing.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_bedrock_pricing.py index 55161d2a7..71533c209 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_bedrock_pricing.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_bedrock_pricing.py @@ -373,6 +373,29 @@ def test_all_five_proprietary_gpt_models_are_priced(): assert v["input_per_1k_usd"] > 0 and v["output_per_1k_usd"] > 0, mid +@pytest.mark.parametrize("model_id", [ + "us.openai.gpt-5.4", "global.openai.gpt-5.4", + "us.openai.gpt-5.5", "global.openai.gpt-5.5", + "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.openai.gpt-5.4", + "arn:aws:bedrock:us-east-1:123456789012:inference-profile/global.openai.gpt-5.5", +]) +def test_gpt54_55_cris_ids_are_unpriced_not_inferred_from_in_region(model_id, monkeypatch): + # PR 583 review: the GPT-5.4 / GPT-5.5 cards (read 2026-11-07) publish commercial + # In-Region prices only. Nothing establishes a Geo/Global CRIS rate, so a CRIS id + # must resolve to `available: false` — never to the bare In-Region row, and never + # via a live PriceList call that cannot carry these models. + import boto3 + + def boom(*a, **k): + raise AssertionError("must not call the PriceList API for a proprietary GPT CRIS id") + + monkeypatch.setattr(boto3, "client", boom) + v = bp.lookup("us-east-1", model_id) + assert v["available"] is False, model_id + assert v["input_per_1k_usd"] is None and v["output_per_1k_usd"] is None + assert "not in the static table" in v["note"] + + def test_all_four_gpt6_models_are_priced(): # Finding 4: GPT-6's bare ids previously had zero STATIC_FALLBACK entries, so # they fell through to the live Pricing API (which never carries OpenAI rows) diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py index b3f6449ab..bd1806ef2 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py @@ -12,10 +12,12 @@ foundation_model_arn, generate_policy, is_gpt6_cris_model, + is_gpt54_55_cris_model, is_mantle_model, mantle_project_arn, inference_profile_arn, is_inference_profile, + requires_default_project_grant, ) REGION = "us-east-1" @@ -267,3 +269,91 @@ def test_bare_gpt6_mantle_id_still_yields_no_invoke_model_statement(self): pol = generate_policy(["openai.gpt-6-sol"], REGION, ACCOUNT) sids = {s["Sid"] for s in pol["Statement"]} assert "BedrockInvokeModelScoped" not in sids + + +_GPT54_55_CRIS_SHORT_IDS = [ + "us.openai.gpt-5.4", + "global.openai.gpt-5.4", + "us.openai.gpt-5.5", + "global.openai.gpt-5.5", +] + + +class TestGpt54Gpt55CrisProjectArn: + """PR 583 review: the GPT-5.4 / GPT-5.5 model cards (read 2026-11-07, + https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html + and ...-gpt-55.html) say the IAM identity "also needs bedrock:InvokeModel on + your account's default project ... in addition to the inference profile" for + their us./global. runtime ids. The grant must cover exactly those ids.""" + + @pytest.mark.parametrize("model_id", _GPT54_55_CRIS_SHORT_IDS) + def test_predicates_match_short_ids(self, model_id): + assert is_gpt54_55_cris_model(model_id) + assert requires_default_project_grant(model_id) + assert not is_gpt6_cris_model(model_id) + + @pytest.mark.parametrize("model_id", _GPT54_55_CRIS_SHORT_IDS) + def test_predicates_match_system_profile_arns(self, model_id): + arn = f"arn:aws:bedrock:{REGION}:{ACCOUNT}:inference-profile/{model_id}" + assert is_gpt54_55_cris_model(arn) + assert requires_default_project_grant(arn) + + @pytest.mark.parametrize( + "model_id", + [ + "openai.gpt-5.4", # bare — mantle, not CRIS + "openai.gpt-5.5", + "us.openai.gpt-5.6-sol", # GPT-5.6 cards do not add the grant + "global.openai.gpt-5.6-terra", + "us.openai.gpt-5.4-pro", # not a published runtime id + "us.anthropic.claude-sonnet-5", + f"arn:aws:bedrock:{REGION}:{ACCOUNT}:inference-profile/us.anthropic.claude-sonnet-5", + ], + ) + def test_predicates_exclude_bare_mantle_gpt56_and_unrelated_ids(self, model_id): + assert not is_gpt54_55_cris_model(model_id) + assert not requires_default_project_grant(model_id) + + @pytest.mark.parametrize("model_id", _GPT54_55_CRIS_SHORT_IDS) + def test_generate_policy_grants_project_default_for_short_id(self, model_id): + pol = generate_policy([model_id], REGION, ACCOUNT) + assert [s["Sid"] for s in pol["Statement"]] == ["BedrockInvokeModelScoped"] + base = model_id.split(".", 1)[1] + assert set(pol["Statement"][0]["Resource"]) == { + f"arn:aws:bedrock:*::foundation-model/{base}", + f"arn:aws:bedrock:{REGION}:{ACCOUNT}:inference-profile/{model_id}", + default_project_arn(REGION, ACCOUNT), + } + + @pytest.mark.parametrize("model_id", _GPT54_55_CRIS_SHORT_IDS) + def test_generate_policy_grants_project_default_for_profile_arn(self, model_id): + arn = f"arn:aws:bedrock:{REGION}:{ACCOUNT}:inference-profile/{model_id}" + pol = generate_policy([arn], REGION, ACCOUNT) + resources = pol["Statement"][0]["Resource"] + assert arn in resources + assert default_project_arn(REGION, ACCOUNT) in resources + + @pytest.mark.parametrize("model_id", ["openai.gpt-5.4", "openai.gpt-5.5"]) + def test_bare_mantle_id_gets_mantle_statements_and_no_project_default(self, model_id): + pol = generate_policy([model_id], REGION, ACCOUNT) + sids = {s["Sid"] for s in pol["Statement"]} + assert "BedrockInvokeModelScoped" not in sids + assert "BedrockMantleInference" in sids + assert default_project_arn(REGION, ACCOUNT) not in json.dumps(pol) + + @pytest.mark.parametrize( + "model_id", + ["us.anthropic.claude-sonnet-5", "us.openai.gpt-5.6-sol", "global.openai.gpt-5.6-luna"], + ) + def test_unrelated_runtime_ids_do_not_get_project_default(self, model_id): + pol = generate_policy([model_id], REGION, ACCOUNT) + resources = pol["Statement"][0]["Resource"] + assert default_project_arn(REGION, ACCOUNT) not in resources + assert len(resources) == 2 + + def test_mixed_run_adds_project_default_once(self): + pol = generate_policy( + ["us.openai.gpt-5.4", "us.anthropic.claude-sonnet-5"], REGION, ACCOUNT + ) + resources = pol["Statement"][0]["Resource"] + assert resources.count(default_project_arn(REGION, ACCOUNT)) == 1 diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md index 78fb426c0..3cc8e290f 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md @@ -43,9 +43,14 @@ Two consequences that invert the previous decision logic: CRIS are priced at OpenAI's _data residency_ tier, exactly 1.10x the standard list price, so those moves cost ~10% more. **GPT-5.6 on Global CRIS is priced at the standard list price — cost parity** — available only when the workload has no data-residency constraint. GPT-5.5 / GPT-5.4 also have `us.`/`global.` CRIS ids on - `bedrock-runtime`, but their model cards (read 2026-11-06) publish In-Region pricing only, so quote the ~10% - figure for them on every inference option until a Global rate is published. See - `shared/openai-on-bedrock.md`. The non-cost case — AWS commitments, IAM/VPC/CloudTrail governance, residency, + `bedrock-runtime`, but their model cards (read 2026-11-07) publish **commercial In-Region prices only** + ("Commercial In-Region prices include a 10% fee over OpenAI rates") — nothing on either card establishes a + Geo or Global CRIS price. So the verified ~10% premium for GPT-5.5 / GPT-5.4 is scoped to the commercial + In-Region (mantle) option. For a GPT-5.5 / GPT-5.4 CRIS plan, either source the Geo/Global rate separately + (the card's inference-option table or [aws.amazon.com/bedrock/pricing](https://aws.amazon.com/bedrock/pricing/), + with URL and read date) or report the comparison unavailable; an In-Region rate used as a proxy is an + **unverified planning assumption**, labelled as such, and yields no verified premium or savings percentage. + Never report parity for them. See `shared/openai-on-bedrock.md`. The non-cost case — AWS commitments, IAM/VPC/CloudTrail governance, residency, prompt caching, one vendor relationship — carries the in-region path; never present that path as free or neutral. For Astra, use its separately dated Bedrock rates and verify the source provider's rate before comparing. @@ -74,8 +79,8 @@ GPT-5.4, the target is **the same model on Bedrock**. | GPT-5.6 Sol | GPT-5.6 Sol | `openai.gpt-5.6-sol` / `us.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | | GPT-5.6 Terra | GPT-5.6 Terra | `openai.gpt-5.6-terra` / `us.` `in.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | | GPT-5.6 Luna | GPT-5.6 Luna | `openai.gpt-5.6-luna` / `us.` `in.` `global.` prefixed | `strong_migrate` — same model; parity on Global CRIS, else +10% | -| GPT-5.5 | GPT-5.5 | `openai.gpt-5.5` / `us.` `global.` prefixed | `strong_migrate` — same model, ~10% over OpenAI std | -| GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` / `us.` `global.` prefixed | `strong_migrate` — same model, ~10% over OpenAI std | +| GPT-5.5 | GPT-5.5 | `openai.gpt-5.5` / `us.` `global.` prefixed | `strong_migrate` — same model; ~10% over OpenAI std In-Region only (no Geo/Global price published) | +| GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` / `us.` `global.` prefixed | `strong_migrate` — same model; ~10% over OpenAI std In-Region only (no Geo/Global price published) | Then apply the **region gate** below. Record `model_change: false` and the chosen path in `aws-design-ai.json` → `ai_architecture.code_migration`: `migration_path: "mantle_openai_chat"` for diff --git a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md index a4206583c..ff4fa58e1 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-pricing-cache.md @@ -133,8 +133,8 @@ See `vendored/ai/ai-model-lifecycle.md` for lifecycle details. **Do not recommen | GPT-5.6 Sol | openai.gpt-5.6-sol | OpenAI | 4.40 | 22.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 8.80/33.00) | | GPT-5.6 Terra | openai.gpt-5.6-terra | OpenAI | 2.20 | 13.20 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 4.40/19.80) | | GPT-5.6 Luna | openai.gpt-5.6-luna | OpenAI | 0.22 | 1.32 | 272K | fast | active (mantle in-region + runtime CRIS; 1M tier 0.44/1.98) | -| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (mantle in-region + runtime CRIS; 1M tier 11.00/49.50; card read 2026-11-06) | -| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (mantle in-region + runtime CRIS; 1M tier 5.50/24.75; card read 2026-11-06) | +| GPT-5.5 | openai.gpt-5.5 | OpenAI | 5.50 | 33.00 | 272K | frontier | active (commercial In-Region price; runtime us./global. CRIS ids exist but carry no published rate; 1M tier 11.00/49.50; card read 2026-11-07) | +| GPT-5.4 | openai.gpt-5.4 | OpenAI | 2.75 | 16.50 | 272K | flagship | active (commercial In-Region price; runtime us./global. CRIS ids exist but carry no published rate; 1M tier 5.50/24.75; card read 2026-11-07) | | Gemma 4 31B | google.gemma-4-31b | Google | 0.14 | 0.40 | 256K | mid | active (Mantle only) | | Gemma 4 26B-A4B | google.gemma-4-26b-a4b | Google | 0.13 | 0.40 | 256K | efficient | active (Mantle only) | | Gemma 4 E2B | google.gemma-4-e2b | Google | 0.04 | 0.08 | 128K | budget | active (Mantle only) | diff --git a/plugins/aws-startup-advisor/tests/test_gpt54_55_runtime_review_round3.py b/plugins/aws-startup-advisor/tests/test_gpt54_55_runtime_review_round3.py new file mode 100644 index 000000000..1ec8aa9dd --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_gpt54_55_runtime_review_round3.py @@ -0,0 +1,164 @@ +"""Guards for the three PR 583 review findings on the GPT-5.4 / GPT-5.5 runtime path. + +Public model cards (read 2026-11-07): + https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html + https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html + +1. IAM: on bedrock-runtime the identity "also needs bedrock:InvokeModel on your + account's default project (arn:aws:bedrock:{region}:{account-id}:project/default) + in addition to the inference profile". The setup recipes must say so for the + GPT-5.5 / GPT-5.4 CRIS ids, not only for GPT-6. +2. Application inference profiles are "Converse API only; not supported with + Responses or Chat Completions APIs". A Design that selects runtime_openai_cris + for that need must carry required_client_api: "converse" and surface a + conflicting Responses-only need as an explicit decision. +3. The cards publish commercial In-Region prices only. No file may infer a Geo or + Global CRIS price (or a premium / savings percentage) for GPT-5.4 / GPT-5.5 from + the In-Region row; an In-Region proxy is an unverified planning assumption. +""" +from __future__ import annotations + +import re +from pathlib import Path + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +SKILLS = PLUGIN_ROOT / "skills" +AGENTS = PLUGIN_ROOT / "agents" + +GCP_GENERATE_AI = SKILLS / "gcp-to-aws" / "references" / "phases" / "generate" / "generate-ai.md" +GCP_GENERATE_ARTIFACTS = SKILLS / "gcp-to-aws" / "references" / "phases" / "generate" / "generate-artifacts-ai.md" +AZURE_GENERATE_ARTIFACTS = SKILLS / "azure-to-aws" / "references" / "phases" / "generate" / "generate-artifacts-ai.md" +GCP_DESIGN_AI = SKILLS / "gcp-to-aws" / "references" / "phases" / "design" / "design-ai.md" +AZURE_DESIGN_AI = SKILLS / "azure-to-aws" / "references" / "phases" / "design" / "design-ai.md" +AZURE_SCHEMA = SKILLS / "azure-to-aws" / "references" / "shared" / "schema-design-aws-ai.md" +REPORT_GENERATOR = AGENTS / "llm2bedrock-report-generator.md" + +CANONICAL_GUIDE = SKILLS / "shared" / "ai" / "ai-openai-to-bedrock.md" +VENDORED_GUIDES = [ + SKILLS / "gcp-to-aws" / "references" / "vendored" / "ai" / "ai-openai-to-bedrock.md", + SKILLS / "azure-to-aws" / "references" / "vendored" / "ai" / "ai-openai-to-bedrock.md", +] +FACT_BASES = [ + SKILLS / "gcp-to-aws" / "references" / "shared" / "openai-on-bedrock.md", + SKILLS / "azure-to-aws" / "references" / "shared" / "openai-on-bedrock.md", +] + +_GPT_54_55 = re.compile(r"GPT-5\.[45]|gpt-5\.[45]", re.IGNORECASE) + + +def _windows(text: str, pattern: re.Pattern[str], radius: int = 700) -> list[str]: + return [text[max(0, m.start() - radius): m.end() + radius] for m in pattern.finditer(text)] + + +# --- Finding 1: default-project grant reaches the GPT-5.5 / GPT-5.4 CRIS ids --- + +def test_setup_recipes_grant_project_default_for_gpt54_55_cris_ids() -> None: + for path in (GCP_GENERATE_AI, GCP_GENERATE_ARTIFACTS, AZURE_GENERATE_ARTIFACTS, REPORT_GENERATOR): + text = path.read_text() + windows = _windows(text, re.compile(r"project/default")) + assert windows, f"{path}: no project/default grant sentence" + assert any(_GPT_54_55.search(w) and "global.openai.gpt-5." in w for w in windows), ( + f"{path}: the project/default grant must name the GPT-5.5 / GPT-5.4 " + "us./global. CRIS ids, not only GPT-6" + ) + # The grant is still a runtime-CRIS grant: the recipe must not hand it to bare mantle ids. + assert not re.search(r"bare `?openai\.gpt-5\.[45]`?[^.]*(gets|needs|requires) (the )?project/default", text), path + + +def test_no_recipe_still_scopes_project_default_to_gpt6_only() -> None: + gpt6_only = re.compile(r"For (a )?GPT-6 CRIS( id)?, (also )?grant `bedrock:InvokeModel` on `arn:aws:bedrock:\{region\}:\{account\}:project/default`\.") + for path in (GCP_GENERATE_AI, GCP_GENERATE_ARTIFACTS): + assert not gpt6_only.search(path.read_text()), f"{path}: project/default grant is still GPT-6-only" + + +# --- Finding 2: application inference profiles force required_client_api: "converse" --- + +_CONVERSE_FIELD = re.compile(r"required_client_api(`)?:?\s*`?\"?converse\"?") + + +def test_design_rule_requires_converse_for_application_profiles_on_gpt54_55() -> None: + for path in (GCP_DESIGN_AI, AZURE_DESIGN_AI, AZURE_SCHEMA): + text = path.read_text() + windows = [w for w in _windows(text, re.compile(r"application inference profiles", re.IGNORECASE)) + if _GPT_54_55.search(w) and _CONVERSE_FIELD.search(w)] + assert windows, ( + f"{path}: no rule ties application inference profiles on GPT-5.5 / GPT-5.4 " + "to required_client_api: \"converse\"" + ) + joined = " ".join(" ".join(w.split()) for w in windows) + assert re.search(r"Converse(-| )only|Converse API only", joined), path + # Generate's absent-field default and the unsupported-pair omission are the failure mode. + assert re.search(r"absent", joined), f"{path}: must forbid leaving required_client_api absent" + # A conflicting Responses-only need is a decision, not a silent drop. + assert re.search(r"explicit (design )?decision", joined), path + assert re.search(r"silent", joined), path + assert re.search(r"structured outputs", joined), path + + +def test_region_only_cris_move_keeps_responses_default() -> None: + for path in (GCP_DESIGN_AI, AZURE_DESIGN_AI, AZURE_SCHEMA): + text = path.read_text() + assert re.search(r"default[^.]*region-only|region-only[^.]*default", text), ( + f"{path}: the responses default for a region-only CRIS move must remain" + ) + + +def test_design_checklists_assert_the_application_profile_rule() -> None: + for path in (GCP_DESIGN_AI, AZURE_SCHEMA): + text = path.read_text() + checklist = text[text.index("## Validation Checklist"):] + item = [w for w in _windows(checklist, re.compile(r"application inference profiles"), 500) + if _CONVERSE_FIELD.search(w) and _GPT_54_55.search(w)] + assert item, f"{path}: Validation Checklist has no application-profile → converse item" + assert "explicit decision" in " ".join(item), path + + +# --- Finding 3: no Geo/Global CRIS price inferred from the In-Region row --- + +_INFERRED_CRIS_PRICE = re.compile( + r"(every inference option|until a Global rate is published|on every inference option)", + re.IGNORECASE, +) + + +def test_no_file_infers_a_cris_price_for_gpt54_55_from_in_region() -> None: + offenders = {} + for path in [CANONICAL_GUIDE, *VENDORED_GUIDES, *FACT_BASES, REPORT_GENERATOR]: + text = path.read_text() + hits = [m.group(0) for m in _INFERRED_CRIS_PRICE.finditer(text) + if _GPT_54_55.search(text[max(0, m.start() - 400): m.end() + 400])] + if hits: + offenders[str(path.relative_to(PLUGIN_ROOT))] = hits + assert not offenders, f"GPT-5.5 / GPT-5.4 CRIS price inferred from the In-Region row: {offenders}" + + +def test_premium_is_scoped_to_commercial_in_region_and_proxy_is_labelled() -> None: + for path in [CANONICAL_GUIDE, *VENDORED_GUIDES, *FACT_BASES, REPORT_GENERATOR]: + text = path.read_text() + windows = [" ".join(w.split()) for w in _windows(text, re.compile(r"In-Region\s+pric(?:es?|ing)\s+(?:only|include)", re.IGNORECASE)) + if _GPT_54_55.search(w)] + assert windows, f"{path}: no GPT-5.5 / GPT-5.4 In-Region-only pricing statement" + joined = " ".join(windows) + assert re.search(r"commercial In-Region", joined, re.IGNORECASE), path + assert re.search(r"Geo( \(`us\.`\))? or Global CRIS price|Geo/Global (rate|price|rows)", joined), path + assert "unverified planning assumption" in joined, path + assert re.search(r"(no|not .*?)(verified )?(premium or savings|premium|savings) percentage", joined), ( + f"{path}: a CRIS plan with only card prices must yield no verified premium/savings percentage" + ) + assert re.search(r"unavailable", joined), f"{path}: must allow reporting the comparison unavailable" + + +def test_tier0_table_scopes_gpt54_55_premium_to_in_region() -> None: + for path in [CANONICAL_GUIDE, *VENDORED_GUIDES]: + rows = [ + line for line in path.read_text().splitlines() + if (line.startswith("| GPT-5.5 ") or line.startswith("| GPT-5.4 ")) and "strong_migrate" in line + ] + assert len(rows) == 2, f"{path}: expected the two Tier 0 GPT-5.5 / GPT-5.4 rows, got {rows}" + for line in rows: + assert "In-Region only" in line and "no Geo/Global price published" in line, line + + +def test_vendored_guides_match_canonical() -> None: + for vendored in VENDORED_GUIDES: + assert vendored.read_bytes() == CANONICAL_GUIDE.read_bytes(), vendored From e39c59a3b00b3e1115a141abf47df4b17641c21e Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 15:20:07 -0800 Subject: [PATCH 07/41] fix(aws-startup-advisor): scope the 1.10x Geo rate to GPT-5.6 GPT-5.4 and GPT-5.5 publish an In-Region price only, so a Geo or Global plan stays unpublished. Co-authored-by: Cursor --- .../phases/clarify/clarify-ai-only.md | 4 +-- .../references/shared/openai-on-bedrock.md | 2 +- .../vendored/ai/ai-openai-to-bedrock.md | 2 +- .../phases/clarify/clarify-ai-only.md | 4 +-- .../phases/clarify/clarify-ai-only.md | 4 +-- .../references/shared/openai-on-bedrock.md | 2 +- .../vendored/ai/ai-openai-to-bedrock.md | 2 +- .../skills/shared/ai/ai-openai-to-bedrock.md | 2 +- .../test_gpt54_55_runtime_review_round3.py | 29 +++++++++++++++++++ 9 files changed, 40 insertions(+), 11 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md index 760e5e5bb..2c3908516 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md @@ -119,8 +119,8 @@ running). → `ai_constraints.cross_cloud`. Default `"latency-acceptable"`. | GPT-6 (Astra/Sol/Luna) / GPT-6.1 Sol | **Same model on Bedrock** | ~10% over OpenAI std (data-residency tier); Astra/6.1 Sol also support Ultrafast (6x) | | GPT-6.1 Terra | **Same model on Bedrock** | Use the model-specific endpoint and context-tier prices in `references/shared/openai-on-bedrock.md`; compare against measured source spend or verified source rates. | | GPT-5.6 (Sol/Terra/Luna) | **Same model on Bedrock** | ~10% over OpenAI std (data-residency tier) | -| GPT-5.5 | **Same model on Bedrock** | ~10% over std | -| GPT-5.4 | **Same model on Bedrock** | ~10% over std | +| GPT-5.5 | **Same model on Bedrock** | In-Region (mantle) ~10% over OpenAI std; no Geo or Global price published | +| GPT-5.4 | **Same model on Bedrock** | In-Region (mantle) ~10% over OpenAI std; no Geo or Global price published | | GPT-4o | GPT-5.6 Terra; or Claude Sonnet 5 | not on Bedrock — offer both | | GPT-4 / 4 Turbo | GPT-5.6 Terra; or Sonnet 5 | not on Bedrock — offer both | | GPT-4.1 / mini / nano | GPT-5.6 Terra/Luna; or Nova Lite/Micro | not on Bedrock — offer both | diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md index 86c2781ab..ffe3ec488 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md @@ -263,7 +263,7 @@ for these models). Sol rates reflect the Aug 21, 2026 reduction (−20% input / the AWS What's New announcement lists as promotional through at least Nov 21, 2026. **Pricing now has an inference-option dimension:** -- **GPT-5.x In-Region and Geo CRIS: 1.10x OpenAI's standard list price** (parity with OpenAI's _data residency_ tier). +- **GPT-5.6 In-Region and Geo CRIS: 1.10x OpenAI's standard list price** (parity with OpenAI's _data residency_ tier). - **Global CRIS: OpenAI's standard list price** — cost parity for the recorded GPT-5.6 rates, and only when the workload has no data-residency constraint. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index 3cc8e290f..dba311abf 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -39,7 +39,7 @@ Two consequences that invert the previous decision logic: 1. **When the source model is on Bedrock, keep it.** A same-model move has no behavior delta to validate, no prompt re-engineering, and no eval regression risk — the migration is an endpoint and credential change, not a model change. This is the default recommendation. -2. **The cost of a same-model move depends on the inference option — state it conditionally.** GPT-5.x In-Region and Geo +2. **The cost of a same-model move depends on the inference option — state it conditionally.** GPT-5.6 In-Region and Geo CRIS are priced at OpenAI's _data residency_ tier, exactly 1.10x the standard list price, so those moves cost ~10% more. **GPT-5.6 on Global CRIS is priced at the standard list price — cost parity** — available only when the workload has no data-residency constraint. GPT-5.5 / GPT-5.4 also have `us.`/`global.` CRIS ids on diff --git a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md index 1b5c2c4e5..7dd9876fb 100644 --- a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md @@ -211,8 +211,8 @@ _Skip when:_ the validator summary reports `models_extractable: true` — every | GPT-6/6.1 (Astra/Sol/Luna) | **Same model on Bedrock** | ~10% over OpenAI std (DR tier); Astra/6.1 Sol also support Ultrafast | | GPT-6.1 Terra | **Same model on Bedrock** | Use the model-specific endpoint and context-tier prices in the [GPT-6.1 Terra model card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-terra.html); compare against measured source spend or verified source rates. | | GPT-5.6 (any) | **Same model on Bedrock** | ~10% over OpenAI std (DR tier) | -| GPT-5.5 | **Same model on Bedrock** | ~10% over OpenAI std (DR tier) | -| GPT-5.4 | **Same model on Bedrock** | ~10% over OpenAI std (DR tier) | +| GPT-5.5 | **Same model on Bedrock** | In-Region (mantle) ~10% over OpenAI std; no Geo or Global price published | +| GPT-5.4 | **Same model on Bedrock** | In-Region (mantle) ~10% over OpenAI std; no Geo or Global price published | | GPT-3.5 Turbo | GPT-5.6 Luna; or Claude Haiku 5.5 | At equal billed volumes and 2:1 input/output, Haiku US Geo is 56.25% cheaper up to 100K prompt tokens; Luna is 54.29% cheaper above 100K while Luna remains within 272K. Recount target tokens; if request size is unknown, present both Haiku tiers. | | GPT-4/4 Turbo | GPT-5.6 Terra; or Sonnet 5 | Not on Bedrock — offer both | | GPT-4o | GPT-5.6 Terra; or Sonnet 5 | Not on Bedrock — offer both | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md index 8634efdc5..bbf691b8c 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md @@ -190,8 +190,8 @@ _Skip when:_ `models[].model_id` is populated in `ai-workload-profile.json` **wi | GPT-6/6.1 (Astra/Sol/Luna) | **Same model on Bedrock** | ~10% over OpenAI std (DR tier); Astra/6.1 Sol also support Ultrafast | | GPT-6.1 Terra | **Same model on Bedrock** | Use the model-specific endpoint and context-tier prices in `references/shared/openai-on-bedrock.md`; compare against measured source spend or verified source rates. | | GPT-5.6 (any) | **Same model on Bedrock** | ~10% over OpenAI std (DR tier) | -| GPT-5.5 | **Same model on Bedrock** | ~10% over OpenAI std (DR tier) | -| GPT-5.4 | **Same model on Bedrock** | ~10% over OpenAI std (DR tier) | +| GPT-5.5 | **Same model on Bedrock** | In-Region (mantle) ~10% over OpenAI std; no Geo or Global price published | +| GPT-5.4 | **Same model on Bedrock** | In-Region (mantle) ~10% over OpenAI std; no Geo or Global price published | | GPT-3.5 Turbo | GPT-5.6 Luna; or Claude Haiku 5.5 | At equal billed volumes and 2:1 input/output, Haiku US Geo is 56.25% cheaper up to 100K prompt tokens; Luna is 54.29% cheaper above 100K while Luna remains within 272K. Recount target tokens; if request size is unknown, present both Haiku tiers. | | GPT-4/4 Turbo | GPT-5.6 Terra; or Sonnet 5 | Not on Bedrock — offer both | | GPT-4o | GPT-5.6 Terra; or Sonnet 5 | Not on Bedrock — offer both | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md index beaeb79b7..b669556ad 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md @@ -268,7 +268,7 @@ for these models). Sol rates reflect the Aug 21, 2026 reduction (−20% input / the AWS What's New announcement lists as promotional through at least Nov 21, 2026. **Pricing now has an inference-option dimension:** -- **GPT-5.x In-Region and Geo CRIS: 1.10x OpenAI's standard list price** (parity with OpenAI's _data residency_ tier). +- **GPT-5.6 In-Region and Geo CRIS: 1.10x OpenAI's standard list price** (parity with OpenAI's _data residency_ tier). - **Global CRIS: OpenAI's standard list price** — cost parity for the GPT-5.6 rates verified above, and only when the workload has no data-residency constraint. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index 3cc8e290f..dba311abf 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -39,7 +39,7 @@ Two consequences that invert the previous decision logic: 1. **When the source model is on Bedrock, keep it.** A same-model move has no behavior delta to validate, no prompt re-engineering, and no eval regression risk — the migration is an endpoint and credential change, not a model change. This is the default recommendation. -2. **The cost of a same-model move depends on the inference option — state it conditionally.** GPT-5.x In-Region and Geo +2. **The cost of a same-model move depends on the inference option — state it conditionally.** GPT-5.6 In-Region and Geo CRIS are priced at OpenAI's _data residency_ tier, exactly 1.10x the standard list price, so those moves cost ~10% more. **GPT-5.6 on Global CRIS is priced at the standard list price — cost parity** — available only when the workload has no data-residency constraint. GPT-5.5 / GPT-5.4 also have `us.`/`global.` CRIS ids on diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md index 3cc8e290f..dba311abf 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md @@ -39,7 +39,7 @@ Two consequences that invert the previous decision logic: 1. **When the source model is on Bedrock, keep it.** A same-model move has no behavior delta to validate, no prompt re-engineering, and no eval regression risk — the migration is an endpoint and credential change, not a model change. This is the default recommendation. -2. **The cost of a same-model move depends on the inference option — state it conditionally.** GPT-5.x In-Region and Geo +2. **The cost of a same-model move depends on the inference option — state it conditionally.** GPT-5.6 In-Region and Geo CRIS are priced at OpenAI's _data residency_ tier, exactly 1.10x the standard list price, so those moves cost ~10% more. **GPT-5.6 on Global CRIS is priced at the standard list price — cost parity** — available only when the workload has no data-residency constraint. GPT-5.5 / GPT-5.4 also have `us.`/`global.` CRIS ids on diff --git a/plugins/aws-startup-advisor/tests/test_gpt54_55_runtime_review_round3.py b/plugins/aws-startup-advisor/tests/test_gpt54_55_runtime_review_round3.py index 1ec8aa9dd..e45e852fe 100644 --- a/plugins/aws-startup-advisor/tests/test_gpt54_55_runtime_review_round3.py +++ b/plugins/aws-startup-advisor/tests/test_gpt54_55_runtime_review_round3.py @@ -121,6 +121,35 @@ def test_design_checklists_assert_the_application_profile_rule() -> None: ) +def test_the_1_10x_geo_rate_names_gpt56_not_the_gpt5x_family() -> None: + """The GPT-5.4 and GPT-5.5 cards publish no Geo or Global price. A GPT-5.x + In-Region-and-Geo 1.10x bullet would price those CRIS ids at the In-Region rate. + """ + for path in [CANONICAL_GUIDE, *VENDORED_GUIDES, *FACT_BASES]: + text = path.read_text() + assert "GPT-5.x In-Region and Geo" not in text, path + assert "GPT-5.6 In-Region and Geo" in text, path + + +CLARIFY_AI_ONLY = [ + SKILLS / "gcp-to-aws" / "references" / "phases" / "clarify" / "clarify-ai-only.md", + SKILLS / "azure-to-aws" / "references" / "phases" / "clarify" / "clarify-ai-only.md", + SKILLS / "bedrock-migration-assessment" / "references" / "phases" / "clarify" / "clarify-ai-only.md", +] + + +def test_clarify_ai_only_scopes_the_gpt54_55_premium_to_in_region() -> None: + for path in CLARIFY_AI_ONLY: + rows = [ + line for line in path.read_text().splitlines() + if re.match(r"\| GPT-5\.5\s+\|", line) or re.match(r"\| GPT-5\.4\s+\|", line) + ] + assert len(rows) == 2, f"{path}: expected two GPT-5.5 / GPT-5.4 rows, got {rows}" + for line in rows: + assert "In-Region (mantle)" in line, line + assert "no Geo or Global price published" in line, line + + def test_no_file_infers_a_cris_price_for_gpt54_55_from_in_region() -> None: offenders = {} for path in [CANONICAL_GUIDE, *VENDORED_GUIDES, *FACT_BASES, REPORT_GENERATOR]: From 810e2ea6cd3b61fcd7845a279609e67929ce2f34 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 7 Nov 2026 09:32:58 -0800 Subject: [PATCH 08/41] fix(aws-startup-advisor): state the bedrock-runtime default endpoint policy and make Runtime the primary same-model OpenAI path in design guidance Base: 6b873f8c (PR 1 #583 tip). Public basis: Amazon Bedrock User Guide endpoints.html, models-endpoint-availability.html, inference-responses-api.html, inference-chat-completions.html, inference-messages-api.html, tool-use-server-side.html, web-search.html, projects.html, workspaces.html, cost-mgmt-application-inference-profiles.html, model-invocation-logging.html, monitoring-mantle-metrics.html, quotas-runtime.html, quotas-mantle.html, scaling-throughput-best-practices.html (all read 2026-11-07). - New canonical skills/shared/ai/bedrock-endpoint-policy.md (rule, mantle_selector enum, public matrix, lifecycle and quota wording), vendored into gcp-to-aws and azure-to-aws (README rows + sync). - ai-openai-to-bedrock.md / ai-openrouter-to-bedrock.md: runtime_openai_cris with the source app's required_client_api is the Tier 0 default; mantle_* only with a mantle_selector; Runtime steps shown first; "smallest change" phrasing removed. - Three schema-design-aws-ai.md copies: runtime_messages, messages, same_model_runtime, mantle_selector vocabulary; Runtime default stated. - gcp/azure design-ai.md, azure design.md/generate.md _assert, gcp/azure generate-artifacts-ai.md, gcp generate-ai.md, gcp estimate.md, azure estimate-assemble.md, both openai-on-bedrock.md Client setup: Runtime first and labelled default, Mantle selector-gated, same_model_runtime route added. - Fixtures: mantle_selector on the two mantle_openai_responses rows. - Tests: new tests/test_endpoint_policy_contract.py; extended test_generate_mantle_dispatch.py, test_ai_only_option_c_framework_route.py, test_gpt6_review_round2.py, test_gpt6_round4_findings.py. Verification (from plugins/aws-startup-advisor): - python3 tools/sync-vendored.py --check -> PASS, 64 files - uv run --no-project --with pytest python -m pytest tests -q -> 1406 passed (base 6b873f8c: 1398) - uv run --project skills/llm-to-bedrock/scripts --with pytest python -m pytest skills/llm-to-bedrock/scripts -q -> 418 passed (unchanged) - uv run --project skills/agent-advisor/scripts --with pytest python -m pytest skills/agent-advisor/scripts -q -> 570 passed (unchanged) - mise run lint:md (0 errors / 823 files), lint:skill-refs (1508 refs, 0 missing), lint:artifacts (99/0), lint:vendored-parity (64), lint:model-id -> PASS - grep -rn -i -E 'primary path|smallest (possible )?change|remains the default| Mantle is the PRIMARY' skills | grep -v vendored -> every Mantle-related hit gone; remaining hits are pre-existing, unrelated to Bedrock endpoints (design-ref-harness.md:29, heroku-to-aws, gcp elastic-beanstalk.md, prompt-library bedrock-quota-manager.md, agent-advisor test_unit_grouping.py). agents/llm2bedrock-code-rewriter.md:260 is left for the execution-lane change. --- .../multi-provider-direct/aws-design-ai.json | 1 + .../aws-design-ai.json | 1 + .../references/phases/design/design-ai.md | 38 +++--- .../references/phases/design/design.md | 2 +- .../phases/estimate/estimate-assemble.md | 5 +- .../phases/generate/generate-artifacts-ai.md | 20 +-- .../references/phases/generate/generate.md | 2 +- .../references/shared/openai-on-bedrock.md | 29 ++++- .../references/shared/schema-design-aws-ai.md | 53 +++++--- .../references/vendored/README.md | 1 + .../vendored/ai/ai-openai-to-bedrock.md | 116 +++++++++++------- .../vendored/ai/ai-openrouter-to-bedrock.md | 6 +- .../vendored/ai/bedrock-endpoint-policy.md | 115 +++++++++++++++++ .../vendored/ai/schema-design-aws-ai.md | 46 +++++-- .../references/phases/design/design-ai.md | 45 ++++--- .../references/phases/estimate/estimate.md | 10 +- .../references/phases/generate/generate-ai.md | 8 +- .../phases/generate/generate-artifacts-ai.md | 27 ++-- .../references/shared/openai-on-bedrock.md | 29 ++++- .../references/shared/schema-design-aws-ai.md | 25 ++++ .../gcp-to-aws/references/vendored/README.md | 1 + .../vendored/ai/ai-openai-to-bedrock.md | 116 +++++++++++------- .../vendored/ai/ai-openrouter-to-bedrock.md | 6 +- .../vendored/ai/bedrock-endpoint-policy.md | 115 +++++++++++++++++ .../vendored/ai/schema-design-aws-ai.md | 46 +++++-- .../skills/shared/ai/README.md | 1 + .../skills/shared/ai/ai-openai-to-bedrock.md | 116 +++++++++++------- .../shared/ai/ai-openrouter-to-bedrock.md | 6 +- .../shared/ai/bedrock-endpoint-policy.md | 115 +++++++++++++++++ .../skills/shared/ai/schema-design-aws-ai.md | 46 +++++-- .../test_ai_only_option_c_framework_route.py | 23 ++++ .../tests/test_endpoint_policy_contract.py | 113 +++++++++++++++++ .../tests/test_generate_mantle_dispatch.py | 26 ++++ .../tests/test_gpt6_review_round2.py | 13 ++ .../tests/test_gpt6_round4_findings.py | 6 + 35 files changed, 1078 insertions(+), 250 deletions(-) create mode 100644 plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md create mode 100644 plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md create mode 100644 plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md create mode 100644 plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py diff --git a/plugins/aws-startup-advisor/fixtures/aws-design-ai/multi-provider-direct/aws-design-ai.json b/plugins/aws-startup-advisor/fixtures/aws-design-ai/multi-provider-direct/aws-design-ai.json index 0330bfc88..e1abf1aab 100644 --- a/plugins/aws-startup-advisor/fixtures/aws-design-ai/multi-provider-direct/aws-design-ai.json +++ b/plugins/aws-startup-advisor/fixtures/aws-design-ai/multi-provider-direct/aws-design-ai.json @@ -93,6 +93,7 @@ "migration_complexity": "low", "model_change": true, "migration_path": "mantle_openai_responses", + "mantle_selector": "in_region_processing", "quota_risk": "low" } ], diff --git a/plugins/aws-startup-advisor/fixtures/aws-design-ai/openrouter-gateway-multi-upstream/aws-design-ai.json b/plugins/aws-startup-advisor/fixtures/aws-design-ai/openrouter-gateway-multi-upstream/aws-design-ai.json index 80da03388..7d03b80a1 100644 --- a/plugins/aws-startup-advisor/fixtures/aws-design-ai/openrouter-gateway-multi-upstream/aws-design-ai.json +++ b/plugins/aws-startup-advisor/fixtures/aws-design-ai/openrouter-gateway-multi-upstream/aws-design-ai.json @@ -157,6 +157,7 @@ "migration_complexity": "medium", "model_change": true, "migration_path": "mantle_openai_responses", + "mantle_selector": "in_region_processing", "quota_risk": "medium" } ], diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md index 0314ff6ca..dd84ff7fe 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md @@ -37,7 +37,8 @@ defaults (prefer managed Bedrock, no latency constraint, no budget cap). **and** `references/shared/openai-on-bedrock.md` (the fact base: Bedrock model IDs, endpoint paths, region matrix, quotas). Azure OpenAI serves OpenAI models, so the source model is usually itself the target — the mapping guide's Tier 0 (same-model) path is the default, not - a cross-family swap. `azure_openai` routes IDENTICALLY to `openai`; the distinct label is + a cross-family swap, and it runs on `bedrock-runtime` (`migration_path: "runtime_openai_cris"`) + unless a `mantle_selector` from `references/vendored/ai/bedrock-endpoint-policy.md` is recorded. `azure_openai` routes IDENTICALLY to `openai`; the distinct label is kept only for report wording and provenance (§19.9(b)). - `"anthropic"` → load `references/vendored/ai/ai-anthropic-to-bedrock.md` (Anthropic SDK → Bedrock Converse client swap). @@ -190,7 +191,8 @@ Partial/None. ## Part 3: Integration pattern → effort Map `integration.pattern`: `direct_sdk` (Azure OpenAI/OpenAI, model on Bedrock, region carries -it) → Mantle Responses API, minimal; `direct_sdk` → Bedrock SDK/Converse, medium; `framework` +it) → Runtime OpenAI API on `bedrock-runtime` CRIS (Mantle only with a `mantle_selector`), minimal; +`direct_sdk` → Bedrock SDK/Converse, medium; `framework` → LangChain/LlamaIndex + Bedrock, low; `rest_api` → Bedrock REST, medium; `mixed` → per-model. ## Part 4: Infrastructure mapping @@ -208,14 +210,20 @@ Map Azure AI infrastructure to AWS: ## Part 5: Code migration plan -Preserve `mantle_openai_chat` when the selected API is Chat Completions. +Preserve the source app's API: a Chat Completions source keeps Chat Completions +(`required_client_api: "chat_completions"`, or `mantle_openai_chat` on a selector-gated Mantle row). Generate before/after examples per detected `integration.pattern` and `ai_source`. For an -`azure_openai`/`openai` source with the model on Bedrock, the **Mantle Responses API** is the -primary path (`migration_path: "mantle_openai_responses"`, `model_change: false`): the app -keeps the OpenAI SDK; only the base URL (`.../openai/v1/responses`), credential (a Bedrock API -key/token provider), model ID, and IAM (`bedrock-mantle:*`) change. Read -`references/shared/openai-on-bedrock.md` for exact values. **A Chat Completions source is a +`azure_openai`/`openai` source with the model on Bedrock, the **`bedrock-runtime` OpenAI-compatible +API** is the default (`migration_path: "runtime_openai_cris"`, `model_change: false`, +`required_client_api` = the API the source already calls): the app keeps the OpenAI SDK; only the +base URL (`https://bedrock-runtime.{region}.amazonaws.com/openai/v1`), credential (a short-term +Bedrock token or API key), model ID (the CRIS profile), and IAM (`bedrock:InvokeModel*`) change. +The **Mantle** path (`migration_path: "mantle_openai_responses"` / `"mantle_openai_chat"`, base URL +`.../openai/v1` on `bedrock-mantle`, `bedrock-mantle:*` IAM) is recorded only when the row also +carries `bedrock_models[].mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`); +API continuity alone never selects Mantle. Read `references/shared/openai-on-bedrock.md` for exact +values. **A Chat Completions source is a supported near-drop-in for GPT-6 and GPT-5.6** (both list Chat Completions on mantle and runtime CRIS per their model cards) — keep `chat.completions.create` unless a Responses-only feature or demonstrated constraint drives a reshape. **GPT-5.5 and GPT-5.4 follow the same rule**: their @@ -228,9 +236,9 @@ Sol/Terra/Luna, all five GPT-6 models (Astra, Sol, Luna, 6.1 Sol, 6.1 Terra), an GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`; Responses, Chat Completions and Converse on runtime, no runtime in-region form) have a `bedrock-runtime` CRIS path (`us.`/`global.` prefixed IDs) — for all of -them a Guardrails/logging/application-inference-profile/unsupported-region need is satisfied by -switching endpoint, not model: record `migration_path: "runtime_openai_cris"` with -`model_change: false`. Reserve `converse` with `model_change: true` to a Bedrock-native model (or +them a Guardrails/logging/application-inference-profile/unsupported-region need is satisfied on +the default endpoint without a model switch: record `migration_path: "runtime_openai_cris"` with +`model_change: false` — the same path a region-only move records. Reserve `converse` with `model_change: true` to a Bedrock-native model (or `gpt-oss`) for a need the same-model CRIS path cannot serve (Knowledge Bases, intelligent prompt routing). Note: `runtime_openai_cris` for GPT-6 Astra is valid only via Responses/Chat Completions/Converse — Astra does not support @@ -240,8 +248,9 @@ Sol/Luna/6.1 Sol. **When Guardrails is the driving need and the design keeps `bedrock_models[].required_client_api` so the generated client uses a Guardrails-capable surface — `converse` for Astra and 6.1 Terra; `converse` / `chat_completions` / `invoke` (never `responses`) for 6.1 Sol and the other GPT-6/5.6 CRIS models except Astra and 6.1 Terra. Do not leave the client on -`responses` when Guardrails is required.** `Responses` stays the default for a region-only -CRIS move with no Guardrails or application-inference-profile need. +`responses` when Guardrails is required.** For a region-only CRIS move with no Guardrails or +application-inference-profile need, the default `required_client_api` is the API the source app +already calls (`responses` or `chat_completions`) — never an implicit Converse. **Application inference profiles are Converse-only on GPT-5.5, GPT-5.4, GPT-6.1 Terra and Astra — the requirement MUST reach `required_client_api`.** The GPT-5.4 / GPT-5.5 model cards (read 2026-11-07) @@ -270,7 +279,8 @@ Keep `model_change: false` only when the exact underlying model is unchanged; op deployment names need verified model identity, not a guessed GPT family. Anthropic-SDK sources → boto3 Converse client swap. Record `code_migration.openrouter_path` -when a router was detected (`same_model_mantle|direct|litellm|keep_openrouter`). +when a router was detected (`same_model_runtime|same_model_mantle|direct|litellm|keep_openrouter`; +`same_model_runtime` is the default same-model route, `same_model_mantle` needs a `mantle_selector`). ## Part 6: Write output diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design.md index 60dda47b0..a7d8a3aab 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design.md @@ -82,7 +82,7 @@ _postconditions: _on_failure: _halt_and_inform - _assert: "WHEN aws-design-ai.json exists: every design_blocks[] row has exactly one of target_bedrock_model / target_aws_service non-null (XOR), and every document_extraction/image_analysis/speech_transcription row has target_bedrock_model null, a non-null target_aws_service, and honest_assessment 'not_applicable' — per references/shared/schema-design-aws-ai.md" _on_failure: _halt_and_inform - - _assert: "WHEN aws-design-ai.json exists AND summary.ai_source is azure_openai or openai: every source model that is available on Bedrock and carried by the target region maps to itself with model_change false — not to a Claude/Nova substitute; and no bare proprietary openai.gpt-* model ID (including bare openai.gpt-5.5 or openai.gpt-5.4) is paired with a converse/bedrock-runtime migration path (bare proprietary openai.gpt-* IDs use Mantle only; runtime_openai_cris accepts documented Astra us./global., GPT-5.6/GPT-6 and GPT-5.5/GPT-5.4 us./global. profiles with caller-region checks and the selected API; GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 models MAY use migration_path: runtime_openai_cris on bedrock-runtime — this is not a violation)" + - _assert: "WHEN aws-design-ai.json exists AND summary.ai_source is azure_openai or openai: every source model that is available on Bedrock and carried by the target region maps to itself with model_change false — not to a Claude/Nova substitute; and no bare proprietary openai.gpt-* model ID (including bare openai.gpt-5.5 or openai.gpt-5.4) is paired with a converse/bedrock-runtime migration path (bare proprietary openai.gpt-* IDs use Mantle only; runtime_openai_cris accepts documented Astra us./global., GPT-5.6/GPT-6 and GPT-5.5/GPT-5.4 us./global. profiles with caller-region checks and the selected API; GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 models MAY use migration_path: runtime_openai_cris on bedrock-runtime — this is not a violation; a same-model proprietary GPT row records migration_path runtime_openai_cris by default and a mantle_* path only with a recorded mantle_selector)" _on_failure: _halt_and_inform - _assert: "WHEN aws-design-ai.json exists: regional_warnings[] and multi_model_warnings[] are present (empty is fine); agentic_design is present only when agentic_profile.is_agentic is true; and App Runner appears nowhere" _on_failure: _halt_and_inform diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate-assemble.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate-assemble.md index 3e8c6c34b..2bb0b9dcd 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate-assemble.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate-assemble.md @@ -121,6 +121,8 @@ Step 3B does — and then show only the matching label: 1. **OpenRouter subroute**, when `aws-design-ai.json` → `code_migration.openrouter_path` is set, or `ai-workload-profile.json` has a `base_url` containing `openrouter.ai`: + - `same_model_runtime` → `[C] Generate the Bedrock runtime (CRIS) migration script and A/B test harness` + (Step 3B skips gateway config and runs Step 1M's CRIS variant; no provider adapter). - `same_model_mantle` → `[C] Generate the Mantle migration script and A/B test harness` (Step 3B skips gateway config and runs Step 1M; no provider adapter). - `direct` → `[C] Generate the Bedrock setup script, provider adapter, and A/B test harness` @@ -128,7 +130,8 @@ Step 3B does — and then show only the matching label: - `litellm`, `keep_openrouter`, or absent → `[C] Generate the gateway config, Bedrock setup script, and A/B test harness` (absent defaults to litellm). Do not promise a provider adapter on these three. 2. **Otherwise** read `ai_architecture.code_migration.migration_path`: - - `mantle_openai_responses` (Design's spelling) or `mantle` → the Mantle label above. + - `runtime_openai_cris` (Design's default) → the Bedrock runtime (CRIS) label above. + - `mantle_openai_responses` / `mantle_openai_chat` (Design's spellings, selector-gated) or `mantle` → the Mantle label above. - `converse`, `direct`, or `gpt-oss` → the adapter label above. `converse` is a Converse client swap, not a Mantle script and not gateway config. - `llm_router` / `api_gateway` / `voice_platform` / `framework` → the gateway diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md index b3f83648e..279af5d92 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -27,8 +27,8 @@ Read `aws-design-ai.json → ai_architecture.code_migration.migration_path` and | `migration_path` / framework | Emits (in addition to the always-on set) | | ----------------------------------------------------------------- | ------------------------------------------------------------------------- | -| `mantle_openai_responses` / `mantle_openai_chat` | `ai-migration/migrate_to_mantle.sh` — **skip the provider adapter** | -| `runtime_openai_cris` | `ai-migration/migrate_to_mantle.sh` (CRIS variant — `bedrock-runtime` base URL + CRIS id) — skip the provider adapter | +| `runtime_openai_cris` (default same-model path) | `ai-migration/migrate_to_mantle.sh` (CRIS variant — `bedrock-runtime` base URL + CRIS id + the row's `required_client_api`) — skip the provider adapter | +| `mantle_openai_responses` / `mantle_openai_chat` (only with `mantle_selector`) | `ai-migration/migrate_to_mantle.sh` — **skip the provider adapter** | | `direct` (or absent) or `converse` | `ai-migration/provider_adapter.{py,js,go}` — `converse` is a Converse client swap, not a Mantle script | | `gpt-oss` | `ai-migration/provider_adapter.{py,js,go}` targeting gpt-oss via Converse | | gateway (`llm_router`/`api_gateway`/`voice_platform`/`framework`) | `ai-migration/gateway_config.{yaml,py,json}` — skip the adapter | @@ -39,10 +39,13 @@ Read `aws-design-ai.json → ai_architecture.code_migration.migration_path` and **Always emit (every path):** `ai-migration/setup_bedrock.sh`, `ai-migration/test_comparison.py` (always Python), `ai-migration/bedrock_monitoring.tf`. -> **Mantle is the PRIMARY path for Azure.** Azure OpenAI apps use the OpenAI SDK, so a same-model -> move keeps the SDK and changes only base URL, credential, model ID, and IAM — `migrate_to_mantle.sh`, -> not a rewritten adapter. The `direct` adapter path is the exception (cross-family swap, or a -> Converse feature the mantle models cannot serve), not the default. +> **`bedrock-runtime` CRIS is the default path for Azure OpenAI.** Azure OpenAI apps use the OpenAI +> SDK, so a same-model move keeps the SDK and changes only base URL +> (`https://bedrock-runtime.{region}.amazonaws.com/openai/v1`), credential, model ID (CRIS profile), +> and IAM (`bedrock:InvokeModel*`) — `migrate_to_mantle.sh` in its CRIS variant, not a rewritten +> adapter. Mantle is used only when Design recorded a `mantle_selector` +> (`references/vendored/ai/bedrock-endpoint-policy.md`). The `direct` adapter path is the exception +> (cross-family swap, or a Converse feature no same-model path can serve), not the default. ## Step 1M: `migrate_to_mantle.sh` (mantle path) @@ -93,8 +96,9 @@ is unaffected. Format by framework: `llm_router` → `gateway_config.yaml` (LiteLLM); `framework` → `gateway_config.py`; `voice_platform` → `gateway_config.json`; `api_gateway` → `gateway_config.yaml`. -OpenRouter branches on `code_migration.openrouter_path` (`same_model_mantle` → run Step 1M; -`direct` → adapter; `litellm` → yaml; `keep_openrouter` → json; absent → litellm). +OpenRouter branches on `code_migration.openrouter_path` (`same_model_runtime` → run Step 1M's CRIS +variant, the default same-model subroute; `same_model_mantle` → run Step 1M's Mantle variant, only +with a `mantle_selector`; `direct` → adapter; `litellm` → yaml; `keep_openrouter` → json; absent → litellm). ## Step 3C / 3E: agentic artifacts diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate.md index 94730ad32..9ed3ca6a6 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate.md @@ -73,7 +73,7 @@ _postconditions: _on_failure: _halt_and_inform - _assert: "no secret VALUE from the inventory appears in any generated artifact; secrets are emitted as Secrets Manager references" _on_failure: _halt_and_inform - - _assert: "WHEN aws-design-ai.json exists AND run_mode is decide_and_execute: generation-ai.json exists and validates, its rollback_plan.mechanism is 'feature_flag' with flag_name 'AI_PROVIDER' and default_value 'azure_openai'; an ai-migration/ directory was produced (setup_bedrock.sh, test_comparison.py, bedrock_monitoring.tf on every path; migrate_to_mantle.sh for the mantle path OR provider_adapter.* for the direct/gpt-oss path, not both); and no bare proprietary openai.gpt-* model ID (including bare openai.gpt-5.5 or openai.gpt-5.4) is paired with a converse/bedrock-runtime path (bare proprietary openai.gpt-* IDs use Mantle only; runtime_openai_cris accepts documented Astra us./global., GPT-5.6/GPT-6 and GPT-5.5/GPT-5.4 us./global. profiles with caller-region checks and the selected API; GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 models MAY use migration_path: runtime_openai_cris on bedrock-runtime — this is not a violation). When aws-design-ai.json is absent this is vacuously satisfied" + - _assert: "WHEN aws-design-ai.json exists AND run_mode is decide_and_execute: generation-ai.json exists and validates, its rollback_plan.mechanism is 'feature_flag' with flag_name 'AI_PROVIDER' and default_value 'azure_openai'; an ai-migration/ directory was produced (setup_bedrock.sh, test_comparison.py, bedrock_monitoring.tf on every path; migrate_to_mantle.sh for the mantle path OR provider_adapter.* for the direct/gpt-oss path, not both); and no bare proprietary openai.gpt-* model ID (including bare openai.gpt-5.5 or openai.gpt-5.4) is paired with a converse/bedrock-runtime path (bare proprietary openai.gpt-* IDs use Mantle only; runtime_openai_cris accepts documented Astra us./global., GPT-5.6/GPT-6 and GPT-5.5/GPT-5.4 us./global. profiles with caller-region checks and the selected API; GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 models MAY use migration_path: runtime_openai_cris on bedrock-runtime — this is not a violation; a same-model proprietary GPT row records migration_path runtime_openai_cris by default and a mantle_* path only with a recorded mantle_selector). When aws-design-ai.json is absent this is vacuously satisfied" _on_failure: _halt_and_inform _forbids_files: - azure-resource-inventory.json diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md index ffe3ec488..0f35e8a4a 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md @@ -174,7 +174,32 @@ path for any GPT-6 model. ### Client setup -Requires the OpenAI SDK at **>= 2.45.0**. Preferred client auto-refreshes a short-term Bedrock token: +Requires the OpenAI SDK at **>= 2.45.0**. The default client targets `bedrock-runtime` (`runtime_openai_cris`) and +auto-refreshes a short-term Bedrock token through a callable `api_key`: + +```python +from aws_bedrock_token_generator import provide_token +from openai import OpenAI + +region = "us-east-1" +client = OpenAI( + base_url=f"https://bedrock-runtime.{region}.amazonaws.com/openai/v1", + api_key=lambda: provide_token(region=region), + max_retries=6, +) + +response = client.responses.create( + model="us.openai.gpt-5.6-terra", # the CRIS inference-profile id, never the bare id, on runtime + input="...", + reasoning={"effort": "medium"}, +) +``` + +Keep the API the source app already calls (`responses.create` or `chat.completions.create`); the runtime endpoint +serves both for these models. + +**Mantle form (only when Design recorded a `mantle_selector`):** the `BedrockOpenAI` client targets +`bedrock-mantle` with the bare model id: ```python from aws_bedrock_token_generator import provide_token @@ -194,7 +219,7 @@ response = client.responses.create( ) ``` -The alternative — `OpenAI(base_url=".../openai/v1", api_key=os.environ["AWS_BEARER_TOKEN_BEDROCK"])` — uses a key +The alternative on either endpoint — `api_key=os.environ["AWS_BEARER_TOKEN_BEDROCK"]` — uses a key that expires within 12 hours and is not refreshed. Do not recommend it for production. **IAM:** on the mantle path, the managed policy `AmazonBedrockMantleInferenceAccess` grants what inference needs, diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md index a78a20e33..ab068ebac 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md @@ -104,15 +104,18 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` `capabilities_matched[]`, `capability_gaps[]`, `honest_assessment`, `source_provider_price`, `bedrock_price`, `price_comparison`, `migration_complexity`, `model_change` (bool), `migration_path` (see below), `quota_risk` (`low|medium|high`), `required_client_api` - (optional — see below). Traditional-AI capabilities do NOT appear here — they are - `design_blocks[]` rows with `target_aws_service`. + (see below), `mantle_selector` (REQUIRED on every `mantle_*` row, absent otherwise — see + below). Traditional-AI capabilities do NOT appear here — they are `design_blocks[]` rows + with `target_aws_service`. - **`capability_mapping`** — per capability that is `true` in the profile's `integration.capabilities_summary`: `{ parity: "full|partial|none", notes }`. - **`code_migration`** — `migration_path` (the selected API/endpoint path consumed by Generate), `primary_pattern` (matches profile `integration.pattern`), `framework`, `files_to_modify[]`, `dependency_changes`, and — when the source is Azure OpenAI / OpenAI — `openrouter_path` when a router was detected - (`same_model_mantle|direct|litellm|keep_openrouter`). + (`same_model_runtime|same_model_mantle|direct|litellm|keep_openrouter`; `same_model_runtime` + is the default same-model route — `bedrock-runtime` CRIS — and `same_model_mantle` requires a + `mantle_selector` on the row). - **`infrastructure[]`** — Azure AI infra resource → AWS equivalent, with confidence. - **`services_to_migrate[]`** — Azure AI service → AWS service, effort, notes. - **`quota_risk`** — optional `low|medium|high` on `ai_architecture`, the value @@ -123,12 +126,22 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` ## migration_path vocabulary -`mantle_openai_chat` preserves a selected Chat Completions API on Mantle for a model whose card -lists it there (GPT-6, GPT-5.5, GPT-5.4). -`mantle_openai_responses` (Azure OpenAI / OpenAI source, model on Bedrock, region carries it — -keep the SDK, `model_change: false`) · `runtime_openai_cris` (GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 -same-model move served via `bedrock-runtime` CRIS — `us.`/`in.`/`global.` prefixed for GPT-5.6 -Terra / Luna, `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4 — `model_change: false`) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set +**`bedrock-runtime` is the default endpoint for every new path** (`vendored/ai/bedrock-endpoint-policy.md`): +a same-model Azure OpenAI / OpenAI source records `runtime_openai_cris`, an Anthropic SDK source +records `runtime_messages`, and a `mantle_*` path is recorded only with a +`bedrock_models[].mantle_selector`. + +`runtime_openai_cris` (GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 same-model move served via +`bedrock-runtime` `/openai/v1` with a CRIS id — `us.`/`in.`/`global.` prefixed for GPT-5.6 +Terra / Luna, `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4 — `model_change: false`, +`required_client_api` = the source app's `responses` or `chat_completions`) · `runtime_messages` +(Anthropic SDK kept on `bedrock-runtime` `/anthropic` with a Claude profile id, +`required_client_api: "messages"`) · `mantle_openai_chat` (preserves a selected Chat Completions +API on Mantle for a model whose card lists it there — GPT-6, GPT-5.5, GPT-5.4; requires +`mantle_selector`) · `mantle_openai_responses` (Azure OpenAI / OpenAI source, model on Bedrock, +region carries it — keep the SDK, `model_change: false`; requires `mantle_selector`) · +`mantle_messages` (Anthropic SDK on the Mantle `/anthropic` endpoint; requires `mantle_selector`, +normally `workspaces`) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, intelligent prompt routing, or another feature the same-model CRIS path itself cannot serve) · `gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK swap). A **bare** @@ -142,11 +155,21 @@ needs Guardrails or Application Inference Profiles (Converse-only features that Invoke-adjacent plumbing on some other models) must verify against Astra's model card per-feature, not assume parity with Sol/Luna/6.1 Sol. -## required_client_api (optional, per `bedrock_models[]` entry) +## mantle_selector (required on every `mantle_*` `bedrock_models[]` entry) + +One of `background_inference | server_side_tools | web_search | projects | workspaces | +mantle_only_model_or_api | in_region_processing` — the verified Mantle-only need that justified +leaving the `bedrock-runtime` default (definitions in `vendored/ai/bedrock-endpoint-policy.md`). +API continuity (the app already calls Responses, Chat Completions or Messages) is not a selector. +Absent on `runtime_*`, `converse`, `gpt-oss` and `direct` rows. + +## required_client_api (per `bedrock_models[]` entry) -`responses | converse | chat_completions | invoke` — records the client API surface the -generated code must use. Absent/default is `responses` for a region-only `runtime_openai_cris` -move (no Guardrails or other client-surface need). A Guardrails-requiring workload on +`responses | converse | chat_completions | invoke | messages` — records the client API surface the +generated code must use. For a region-only `runtime_openai_cris` move (no Guardrails or other +client-surface need) record the API the source app already calls — `responses` for a Responses +source, `chat_completions` for a Chat Completions source; Converse is never an implicit default +for `runtime_openai_cris`. `messages` belongs to `runtime_messages`. A Guardrails-requiring workload on `runtime_openai_cris` sets it to the surface that is actually Guardrails-capable for that model: `converse` for GPT-6 Astra and GPT-6.1 Terra, or `converse`/`chat_completions`/`invoke` — never `responses` — for GPT-6.1 Sol and the other @@ -222,7 +245,9 @@ rubric file (`ai.md`, `ai-azure-openai-to-bedrock.md`, or a `vendored/ai/*` ref) move keeps the `responses` default. - [ ] If `ai_source` is `azure_openai` or `openai`: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not - to a Claude/Nova substitute. + to a Claude/Nova substitute — and records `migration_path: "runtime_openai_cris"` by default. +- [ ] Every `bedrock_models[]` row whose `migration_path` starts with `mantle_` carries a + `mantle_selector` from the enum above; no `runtime_*`/`converse` row carries one. - [ ] `regional_warnings` and `multi_model_warnings` are present (`[]` when clean). - [ ] If `agentic_profile.is_agentic == true`: `agentic_design` is present with `migration_approach` matching `preferences.json`; a `harness` source with an diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md index b0980a9ba..c76edd0de 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/README.md @@ -37,6 +37,7 @@ if this skill needs one, add the copy **and** a row in the table below. | `ai/ai-openai-to-bedrock.md` | `skills/shared/ai/ai-openai-to-bedrock.md` | | `ai/ai-gemini-to-bedrock.md` | `skills/shared/ai/ai-gemini-to-bedrock.md` | | `ai/ai-openrouter-to-bedrock.md` | `skills/shared/ai/ai-openrouter-to-bedrock.md` | +| `ai/bedrock-endpoint-policy.md` | `skills/shared/ai/bedrock-endpoint-policy.md` | | `ai/bedrock-pricing-cache.md` | `skills/shared/ai/bedrock-pricing-cache.md` | | `ai/bedrock-quotas.md` | `skills/shared/ai/bedrock-quotas.md` | | `ai/design-ref-agentic-to-agentcore.md` | `skills/shared/ai/design-ref-agentic-to-agentcore.md` | diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index dba311abf..852c8e2fb 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -83,11 +83,19 @@ GPT-5.4, the target is **the same model on Bedrock**. | GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` / `us.` `global.` prefixed | `strong_migrate` — same model; ~10% over OpenAI std In-Region only (no Geo/Global price published) | Then apply the **region gate** below. Record `model_change: false` and the chosen path in -`aws-design-ai.json` → `ai_architecture.code_migration`: `migration_path: "mantle_openai_chat"` for -sources using Chat Completions on a model whose card lists it on mantle (GPT-6, GPT-5.5, GPT-5.4), or -`"mantle_openai_responses"` for a selected Mantle Responses path. Use `migration_path: "runtime_openai_cris"` -when a GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 target is served via `bedrock-runtime` with a CRIS id (the model -cards recommend runtime for new applications). +`aws-design-ai.json` → `ai_architecture.code_migration`. **The default for every Tier 0 target is +`migration_path: "runtime_openai_cris"`** — the same model served via `bedrock-runtime` with a CRIS +id (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, +GPT-5.5 and GPT-5.4) — with +`bedrock_models[].required_client_api` set to the API the source app already calls (`responses` or +`chat_completions`; `converse` only when a Converse-only feature such as Guardrails is required). +Amazon Bedrock recommends `bedrock-runtime` for new applications +(`references/vendored/ai/bedrock-endpoint-policy.md`). Record `migration_path: "mantle_openai_chat"` +(Chat Completions source on a model whose card lists it on mantle — GPT-6, GPT-5.5, GPT-5.4) or +`"mantle_openai_responses"` **only** when the row also records a `bedrock_models[].mantle_selector` +from that policy file (background inference, server-side tools / Web Search, Projects / Workspaces, +a Mantle-only model or API, or strict in-Region processing where runtime offers only CRIS). API +continuity alone never selects Mantle. Report the assessment honestly: `strong_migrate` here means "low-risk, well-supported move", not "cheaper". @@ -102,26 +110,30 @@ If neither path meets the constraints, offer a supported region or a model chang Do not assume GPT-5.6's broader regional footprint applies to Astra. **GPT-6 Astra / Sol / Luna / 6.1 Sol / 6.1 Terra:** same shape as GPT-5.6 — if the target region is in the mantle in-region matrix (narrower than GPT-5.6's: us-east-1 + us-west-2 for Astra only, us-east-1 only for Sol/Luna/6.1 Sol/6.1 Terra — see -`openai-on-bedrock.md` § Regional Availability), either endpoint works; otherwise the `bedrock-runtime` CRIS path -(`us.`/`global.` prefixed) covers broad commercial regions. GPT-6 Astra's CRIS path additionally lacks `Invoke` +`openai-on-bedrock.md` § Regional Availability), both endpoints work and the `bedrock-runtime` CRIS path +(`us.`/`global.` prefixed) is chosen unless a `mantle_selector` applies; outside that matrix the same CRIS path +covers broad commercial regions. GPT-6 Astra's CRIS path additionally lacks `Invoke` support (Responses/Chat Completions/Converse only) — verify per-feature before assuming parity with Sol/Luna/6.1 Sol. -**GPT-5.6 Sol / Terra / Luna:** rarely blocked. If the target region is in the mantle in-region matrix, either -endpoint works. If not, the `bedrock-runtime` CRIS path (Geo `us.`/`in.`, Global `global.`) covers most commercial -regions — offer it, stating the data-residency implication of cross-region routing (Geo stays within the geography; -Global routes anywhere and is the cost-parity option). Only a workload that requires strictly in-region processing -in a region outside the mantle matrix falls through to Tier 1. +**GPT-5.6 Sol / Terra / Luna:** rarely blocked. The `bedrock-runtime` CRIS path (Geo `us.` — plus `in.` for Terra +and Luna only — and Global `global.`) covers most commercial regions and is the default whether or not the target region is also in the +mantle in-region matrix — state the data-residency implication of cross-region routing (Geo stays within the +geography; Global routes anywhere and is the cost-parity option). A workload that requires strictly in-region +processing in a region inside the mantle matrix records `mantle_selector: "in_region_processing"` and takes the +Mantle path; one that requires it in a region outside the mantle matrix falls through to Tier 1. **GPT-5.5 / GPT-5.4:** same shape as GPT-5.6, with a narrower mantle matrix (GPT-5.5: us-east-1, us-east-2; -GPT-5.4: us-east-1, us-east-2, us-west-2, us-gov-west-1 — per their model cards, read 2026-11-06). If the target -region is in that matrix, either endpoint works. Otherwise the `bedrock-runtime` CRIS path (`us.openai.gpt-5.5` / -`global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`) reaches it, with the same data-residency -caveat as GPT-5.6 (Geo stays within the geography, Global routes anywhere). There is no runtime in-region form. -Only a workload that requires strictly in-region processing in a region outside the mantle matrix falls through: - -1. Prefer offering a **region change** to a supported mantle region, if no data-residency constraint pins the - workload. This preserves the zero-behavior-delta benefit, which is the whole point of Tier 0. +GPT-5.4: us-east-1, us-east-2, us-west-2, us-gov-west-1 — per their model cards, read 2026-11-06). The +`bedrock-runtime` CRIS path (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / +`global.openai.gpt-5.4`) is the default, with the same data-residency caveat as GPT-5.6 (Geo stays within the +geography, Global routes anywhere). There is no runtime in-region form. Strict in-region processing inside the +mantle matrix is a valid `in_region_processing` selector for the Mantle path. Only a workload that requires +strictly in-region processing in a region outside the mantle matrix falls through: + +1. Prefer offering a **region change** to a supported mantle region (recorded with + `mantle_selector: "in_region_processing"`), if no data-residency constraint pins the workload. This preserves + the zero-behavior-delta benefit, which is the whole point of Tier 0. 2. Otherwise fall through to Tier 1 and say why explicitly — a region constraint, not a model judgment. Record the outcome in `regional_warnings[]`. @@ -290,11 +302,11 @@ rather than dismissing it. For a long-context (>272K) workload the gap is much w | OpenAI Feature | Bedrock Equivalent | Notes | | ------------------------------------------ | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | -| OpenAI SDK (direct) | Same model on mantle Responses API | Base URL + credential + model ID change; see `shared/openai-on-bedrock.md` for the `/openai/v1` path | -| Responses API | Mantle Responses API | Closest to a drop-in; still verify the path and model ID | +| OpenAI SDK (direct) | Same model on `bedrock-runtime` `/openai/v1` (CRIS id) | Base URL + credential + model ID change; see `shared/openai-on-bedrock.md` for the `/openai/v1` path; Mantle only with a `mantle_selector` | +| Responses API | Runtime Responses API (`/openai/v1/responses`) | Closest to a drop-in; always synchronous on runtime (`background=true` needs a Mantle selector) | | Chat Completions | Preserve supported Chat when selected | Reshape only for a selected API change; verify the model and endpoint | -| Function calling | Supported on GPT-5.x via mantle; Claude tools via Converse | Same-model keeps tool semantics identical | -| Reasoning effort | `reasoning={"effort": ...}` on mantle | `none` / `low` / `medium` / `high` / `xhigh` / `max` | +| Function calling | Supported on GPT-5.x via runtime or mantle; Claude tools via Converse | Same-model keeps tool semantics identical; server-side tools are Mantle-only | +| Reasoning effort | `reasoning={"effort": ...}` on either endpoint | `none` / `low` / `medium` / `high` / `xhigh` / `max` | | Prompt caching | GPT-5.6 (90% off cached input); GPT-5.5 / 5.4 implicit caching on `bedrock-runtime` Responses only | GPT-5.5 / 5.4 cards (read 2026-11-06) list a cache-read rate and implicit caching on runtime, none on mantle; Claude has its own caching | | Streaming | Supported | Verify per surface | | Vision | GPT-5.x (image input) or Claude / Llama 4 | Same-model preserves behavior | @@ -306,22 +318,34 @@ rather than dismissing it. For a long-context (>272K) workload the gap is much w | Assistants API | See decision tree below | Path depends on which features are used | | Realtime API | No equivalent | Stay on OpenAI for this | | Codex | Not verified on Bedrock — see `shared/openai-on-bedrock.md` | Do not price; re-check the OpenAI model card index first | -| Guardrails / KB / invocation logging | Bedrock-native model or gpt-oss | Not available through the mantle GPT path | +| Guardrails / KB / invocation logging | `bedrock-runtime` CRIS path (Guardrails via Converse; logging on every runtime API); KB needs a Bedrock-native model or gpt-oss | None of these are available through the mantle GPT path | --- ## Common Migration Paths -### Same model on Bedrock (Tier 0) — smallest possible change +### Same model on Bedrock (Tier 0) — `bedrock-runtime` by default -The application keeps the OpenAI SDK and the same model. Changes are limited to: +The application keeps the OpenAI SDK, the same model and the same API it calls today +(`migration_path: "runtime_openai_cris"`, `required_client_api` = `responses` or `chat_completions`). Changes are +limited to: -1. Base URL → `https://bedrock-mantle.{region}.api.aws/openai/v1` (note the `openai/v1` segment) -2. Credentials → Bedrock API key or the auto-refreshing `BedrockOpenAI` token provider, **not** an OpenAI key -3. Model ID → the `openai.gpt-*` form -4. IAM → `bedrock-mantle:*` actions, e.g. via `AmazonBedrockMantleInferenceAccess` +1. Base URL → `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` (note the `openai/v1` segment) +2. Credentials → a short-term Bedrock bearer token (`api_key=lambda: provide_token(region=region)` from + `aws-bedrock-token-generator`) or a Bedrock API key, **not** an OpenAI key +3. Model ID → the CRIS inference-profile id (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only + for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), never the bare `openai.gpt-*` form +4. IAM → `bedrock:InvokeModel*` on the inference-profile and foundation-model ARNs (GPT-6 / GPT-5.5 / GPT-5.4 + also need `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account-id}:project/default`), plus + `bedrock:GetInferenceProfile` on the profile -Full snippet and prerequisites in `shared/openai-on-bedrock.md`. No prompt changes, no eval regression expected — +**Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` +(`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, +the auto-refreshing `BedrockOpenAI` token provider or a Bedrock API key, the bare `openai.gpt-*` id, and +`bedrock-mantle:*` IAM (e.g. `AmazonBedrockMantleInferenceAccess`). Record `migration_path: "mantle_openai_responses"` +or `"mantle_openai_chat"` with the selector. + +Full snippets and prerequisites in `shared/openai-on-bedrock.md`. No prompt changes, no eval regression expected — but still run the eval harness to confirm, because infrastructure and tokenizer-adjacent behavior can differ. ### Assistants API → migration decision tree @@ -329,16 +353,17 @@ but still run the eval harness to confirm, because infrastructure and tokenizer- Assistants API and Responses API are different surfaces. Do not treat all Assistants usage as a config-only change. **1. App already uses the OpenAI Responses API** (`responses.create`) -→ Cleanest path. Base URL + credential + model ID change. +→ Cleanest path. Base URL + credential + model ID change on `bedrock-runtime` (`runtime_openai_cris`). **2. App uses Assistants API only for stateful multi-turn conversation** (no hosted tools, no file search, no code interpreter, no persistent Assistant objects, no complex run lifecycle) -→ Mantle Responses is viable. Requires migrating `threads`/`runs` to `responses.create` — a small API migration -(days), not a redesign. Not a zero-code change. +→ Runtime Responses is viable (stateful `previous_response_id` works on `bedrock-runtime`). Requires migrating +`threads`/`runs` to `responses.create` — a small API migration (days), not a redesign. Not a zero-code change. **3. App uses Assistants API with simple hosted tools** (function calling only) -→ Mantle Responses with tool use is viable. Moderate migration (1–2 weeks) to adapt tool definitions and the run -lifecycle. +→ Runtime Responses with client-side tool use is viable. Moderate migration (1–2 weeks) to adapt tool definitions +and the run lifecycle. Server-side or pre-configured tools (including Web Search) are a `server_side_tools` / +`web_search` Mantle selector. **4. App uses Assistants API with file search, vector stores, code interpreter, persistent Assistant objects, or complex run lifecycle management** @@ -347,15 +372,16 @@ Gateway, gateway-fronted knowledge bases) for a full agentic replacement (2–4 if the team prefers to own state. Never target classic Bedrock Agents (`bedrock-agent`) — it is in maintenance mode and closed to new customers as of July 30, 2026. -### When to prefer a Bedrock-native model over the mantle GPT path +### When to prefer a Bedrock-native model over the same-model GPT path -Choose Claude / Nova via Converse when the workload needs Knowledge Bases, intelligent prompt routing, structured -outputs, or application inference profiles — none of which the mantle GPT path offers. Note what no longer forces -a family switch for GPT-5.6, GPT-5.5 and GPT-5.4: their `bedrock-runtime` CRIS paths provide cross-region -inference, invocation logs, Guardrails and application inference profiles (both Converse API only), and -structured outputs and implicit prompt caching (both Responses API only) — per the GPT-5.4 / GPT-5.5 model cards -read 2026-11-06. Knowledge Bases and intelligent prompt routing still require a Bedrock-native model. The mantle -GPT path is the smallest change; the Bedrock-native path remains the most feature-complete. +Choose Claude / Nova via Converse when the workload needs Knowledge Bases or intelligent prompt routing — neither +is served by a same-model GPT path on either endpoint. Note what no longer forces a family switch for GPT-5.6, +GPT-5.5 and GPT-5.4: their `bedrock-runtime` CRIS paths (the default) provide cross-region inference, invocation +logs, Guardrails and application inference profiles (both Converse API only), and structured outputs and implicit +prompt caching (both Responses API only) — per the GPT-5.4 / GPT-5.5 model cards read 2026-11-06. Knowledge +Bases and intelligent prompt routing still require a Bedrock-native model. The same-model `bedrock-runtime` path +is the default; the Bedrock-native path is the most feature-complete; the mantle GPT path is reserved for a +recorded `mantle_selector`. ### High-spend tiering diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openrouter-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openrouter-to-bedrock.md index ca91e4173..18ff0e281 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openrouter-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openrouter-to-bedrock.md @@ -62,7 +62,7 @@ For every distinct upstream model behind the gateway: 1. **Select the provider guide** from the provider segment (`anthropic` / `openai` / `google`). 2. **Apply that guide's selection policy** to the exact model segment — same-model-on-Bedrock where it exists (OpenAI's Tier 0, Claude version-to-version), else the guide's cross-family / tier mapping. The provider guides own all pricing tables, region gates, and lifecycle checks; do not restate them here. -3. **Record the mapping per upstream, including the guide's selected endpoint** — the gateway's output is a set of `{ upstream_model → bedrock_target }` decisions, not one decision, and each decision carries the `migration_path`, endpoint/inference-profile ID, and credential/IAM requirement the provider guide chose. For GPT targets `ai-openai-to-bedrock.md` may select the Mantle Responses path (`mantle_openai_responses`) **or** the `bedrock-runtime` CRIS path (`runtime_openai_cris`, with a `us.`/`global.` profile) depending on the model and region gate — carry whichever it selected through to cutover, do not collapse every GPT target onto Mantle. Two Claude models behind the same gateway produce two rows, never one. +3. **Record the mapping per upstream, including the guide's selected endpoint** — the gateway's output is a set of `{ upstream_model → bedrock_target }` decisions, not one decision, and each decision carries the `migration_path`, endpoint/inference-profile ID, and credential/IAM requirement the provider guide chose. For GPT targets `ai-openai-to-bedrock.md` selects the `bedrock-runtime` CRIS path by default (`runtime_openai_cris`, with a `us.`/`global.` profile and the source app's `required_client_api`) and the Mantle path (`mantle_openai_responses` / `mantle_openai_chat`) only with a recorded `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`) — carry whichever it selected through to cutover, do not collapse every GPT target onto Mantle. Two Claude models behind the same gateway produce two rows, never one. Preserve the distinction between a provider reached **through OpenRouter** and the **same provider reached directly**. A workload may use direct Anthropic SDK calls *and* Anthropic-via-OpenRouter at once; both map to Bedrock-hosted Claude, but they are different source integrations with different cutover work (a direct SDK swap vs. removing a gateway hop). @@ -72,8 +72,8 @@ Preserve the distinction between a provider reached **through OpenRouter** and t Replacing OpenRouter with Bedrock is an integration change, not just a model change: -- **Transport — per the selected `migration_path`, not one default.** For Claude / Nova / Llama / Gemini-replacement targets, the OpenRouter client becomes the Bedrock Converse API. For GPT targets the transport follows the provider guide's selection: the Bedrock Mantle OpenAI-compatible endpoint when it chose `mantle_openai_responses`, or `bedrock-runtime` with the selected CRIS inference profile when it chose `runtime_openai_cris`. Do not force every GPT target onto Mantle — a runtime/CRIS selection sent to Mantle (or a Mantle-only model routed to runtime) is an incompatible cutover plan. A workload that used OpenRouter precisely for its one-API-many-models convenience may want a thin internal router over Bedrock to preserve that shape — see `references/vendored/ai/ai-migration-guardrails.md`. -- **Credentials.** Remove the OpenRouter key and any BYOK upstream keys; use IAM matched to each target's selected path — `bedrock:InvokeModel` for Converse and the `runtime_openai_cris` path, `bedrock-mantle:*` (e.g. `AmazonBedrockMantleInferenceAccess`) for the Mantle GPT path. The source-key retirement rules apply — do not plant the old gateway/provider keys in the target. +- **Transport — per the selected `migration_path`; the default is `bedrock-runtime`.** For Claude / Nova / Llama / Gemini-replacement targets, the OpenRouter client becomes the Bedrock Converse API. For GPT targets the transport follows the provider guide's selection: `bedrock-runtime` `/openai/v1` with the selected CRIS inference profile when it chose `runtime_openai_cris` (the default — recorded as `openrouter_path: "same_model_runtime"`), or the Bedrock Mantle OpenAI-compatible endpoint only when it chose `mantle_openai_responses` / `mantle_openai_chat` with a `mantle_selector` (`openrouter_path: "same_model_mantle"`). Do not force every GPT target onto Mantle — a runtime/CRIS selection sent to Mantle (or a Mantle-only model routed to runtime) is an incompatible cutover plan. A workload that used OpenRouter precisely for its one-API-many-models convenience may want a thin internal router over Bedrock to preserve that shape — see `references/vendored/ai/ai-migration-guardrails.md`. +- **Credentials.** Remove the OpenRouter key and any BYOK upstream keys; use IAM matched to each target's selected path — `bedrock:InvokeModel*` for Converse and the `runtime_openai_cris` path (the default), `bedrock-mantle:*` (e.g. `AmazonBedrockMantleInferenceAccess`) only for a selector-gated Mantle GPT path. The source-key retirement rules apply — do not plant the old gateway/provider keys in the target. - **Routing policy.** If the app used OpenRouter's automatic model selection or fallback routing, that policy has no Bedrock equivalent and must be reimplemented in the application or dropped deliberately — flag it rather than assuming parity. - **Per-upstream validation.** Validate each mapped target independently (behavior, tool-calling, structured output) the way its provider guide prescribes; a gateway migration is only complete when every retained upstream passes. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md new file mode 100644 index 000000000..fdc4948a2 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md @@ -0,0 +1,115 @@ +# Bedrock Endpoint Policy — `bedrock-runtime` by default, `bedrock-mantle` by selector + +> Canonical endpoint-selection policy for every Bedrock migration path in this plugin. +> Vendored into each consuming skill as `references/vendored/ai/bedrock-endpoint-policy.md` +> and kept byte-identical by `tools/sync-vendored.py`; edit HERE, then run the sync. +> Every statement below comes from the public Amazon Bedrock User Guide pages listed in +> § Sources (read 2026-11-07). Re-read those pages before changing a row. + +## The rule + +**Use `bedrock-runtime` for new applications.** Amazon Bedrock documents `bedrock-runtime` as the +recommended endpoint for most new applications; it serves InvokeModel, Converse, the +OpenAI-compatible Responses and Chat Completions APIs, and the Anthropic Messages API. + +**Use `bedrock-mantle` only for a verified Mantle-only need.** The documented reasons are: + +- background (asynchronous, long-running) inference — Responses `background=true` is rejected + with a 400 on `bedrock-runtime`; +- server-side tool use and pre-configured tools, including Web Search; +- customer-created Projects (OpenAI-compatible) or Workspaces (Anthropic-compatible) — + `bedrock-runtime` exposes only the account's default project; +- a model or API that the Endpoint availability page lists on `bedrock-mantle` only; +- strict in-Region processing for a model that `bedrock-runtime` serves only through a + cross-Region inference (CRIS) profile while `bedrock-mantle` serves it in the required Region. + +**API continuity alone never selects Mantle.** An OpenAI Responses, OpenAI Chat Completions or +Anthropic Messages source keeps its SDK on `bedrock-runtime` by changing the base URL, the +credential and the model ID. Choosing Mantle because "the app already speaks Responses" is not a +selector. + +**Pricing never selects an endpoint.** Per-token pricing for the same model is identical on both +endpoints; choose by API and capability need. + +## `mantle_selector` — the recorded reason + +Every `bedrock_models[]` row whose `migration_path` starts with `mantle_` (`mantle_openai_responses`, +`mantle_openai_chat`, `mantle_messages`) MUST record `mantle_selector` with exactly one value: + +| `mantle_selector` | Verified need | +| --------------------------- | ----------------------------------------------------------------------------------------------- | +| `background_inference` | Responses requests with `background=true` / long-running asynchronous inference | +| `server_side_tools` | Server-side tool use (Lambda-backed MCP tools) or pre-configured tools other than Web Search | +| `web_search` | The built-in Web Search tool | +| `projects` | Customer-created Projects (OpenAI-compatible) for isolation and cost tracking | +| `workspaces` | Workspaces (Anthropic-compatible) referenced by `anthropic-workspace-id` | +| `mantle_only_model_or_api` | The model, or the API for that model, is listed on `bedrock-mantle` only | +| `in_region_processing` | Strict in-Region processing where `bedrock-runtime` offers only a CRIS profile for that model | + +A `mantle_*` row without a `mantle_selector`, or with a reason outside this table, is a design +defect: rewrite the row as the `bedrock-runtime` path (`runtime_openai_cris` with the source +app's `required_client_api`, or `runtime_messages`) or justify it with one of the values above. +A `runtime_*` or `converse` row never carries `mantle_selector`. + +## Public comparison (read 2026-11-07) + +| Item | `bedrock-runtime` | `bedrock-mantle` | Source page | +| ---------------------------------------------- | --------------------------------------------------------------------------------- | --------------------------------------------------------------- | ----------------------------------------------- | +| Recommended default | Yes — "for most new applications" | Only for capabilities available only there | `endpoints.html`, `models-endpoint-availability.html` | +| InvokeModel / Converse | Supported | Not supported | `endpoints.html` | +| OpenAI Responses API | Supported at `/openai/v1`; always synchronous; default project only | Supported at `/v1` (some models `/openai/v1`); background OK | `inference-responses-api.html` | +| OpenAI Chat Completions API | Supported at `/openai/v1/chat/completions` | Supported at `/v1/chat/completions` (some models `/openai/v1`) | `inference-chat-completions.html` | +| Anthropic Messages API | Supported at `/anthropic` (SDK sends `/anthropic/v1/messages`) | Supported at `/anthropic/v1/messages` | `inference-messages-api.html` | +| Background Responses (`background=true`) | Rejected (400) | Supported | `endpoints.html`, `inference-responses-api.html` | +| Server-side tool use / pre-configured tools | Not available; client-side tools work | Supported | `tool-use-server-side.html`, `endpoints.html` | +| Web Search | Not available | Supported | `web-search.html`, `endpoints.html` | +| Projects / Workspaces | Default project only; use inference profiles for isolation and tagging | Create and manage your own | `projects.html`, `workspaces.html` | +| Cross-Region inference (Geo / Global profiles) | Supported | Not supported — in-Region only | `endpoints.html`, `monitoring-mantle-metrics.html` | +| In-Region processing | Through an on-demand (unprefixed) model ID where the model card lists one; proprietary GPT models list CRIS profiles only | Always in-Region | `endpoints.html`, the per-model cards | +| Guardrails, intelligent prompt routing | Supported | Not supported | `endpoints.html` | +| Application inference profiles / per-request metadata | InvokeModel and Converse only (Responses and Chat Completions reject an AIP target with 400) | Not available — Projects / Workspaces attribute usage | `cost-mgmt-application-inference-profiles.html`, `endpoints.html` | +| Model invocation logging | Supported, including the OpenAI-compatible APIs on this endpoint | Not currently captured | `model-invocation-logging.html` | +| CloudWatch metrics namespace | `AWS/Bedrock` (`Invocations`, `InputTokenCount`, …) | `AWS/BedrockMantle` (`Inferences`, `TotalInputTokens`, …) | `monitoring-mantle-metrics.html` | +| Authentication | SigV4 or Bedrock API key / short-term bearer token; IAM `bedrock:InvokeModel*` | SigV4 or Bedrock API key; IAM `bedrock-mantle:CreateInference` | `endpoints.html` | + +## Lifecycle wording (the only sentence to repeat) + +Existing `bedrock-mantle` applications continue to be fully supported and do not need to change. +Do not add claims about permanence, model-quality equivalence between the endpoints, +launch-day parity for new models, or future plans — the public pages make none, and the +guides in this plugin must not either. Both endpoint surfaces use the same underlying inference +engine (`endpoints.html`). + +## Quotas (why endpoint choice affects capacity planning) + +- **`bedrock-runtime`** — per-model, per-Region token quotas (input and output counted + together, output weighted by a model-specific burndown rate) shared across every inference + API on the endpoint; a requests-per-minute quota exists for some models and not others + (`quotas-runtime.html`, `scaling-throughput-best-practices.html`). +- **`bedrock-mantle`** — no requests-per-minute quota; separate input-tokens-per-minute and + output-tokens-per-minute quotas per model and Region for models with published quotas. A + model without a published quota is not unlimited — internal service capacity and transient + rate limiting still apply (`quotas-mantle.html`, `scaling-throughput-best-practices.html`). +- Traffic on the two endpoints consumes separate quota allocations even for the same model; + plan capacity per endpoint. Batch inference, Provisioned Throughput and custom inference + profiles exist only on `bedrock-runtime`. + +## Sources + +All pages under `https://docs.aws.amazon.com/bedrock/latest/userguide/`, read 2026-11-07: + +- `endpoints.html` — https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html +- `models-endpoint-availability.html` +- `inference-responses-api.html` +- `inference-chat-completions.html` +- `inference-messages-api.html` +- `tool-use-server-side.html` +- `web-search.html` +- `projects.html` +- `workspaces.html` +- `cost-mgmt-application-inference-profiles.html` +- `model-invocation-logging.html` +- `monitoring-mantle-metrics.html` +- `quotas-runtime.html` +- `quotas-mantle.html` +- `scaling-throughput-best-practices.html` diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md index c82a8eebc..820592554 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -175,26 +175,41 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` - **`bedrock_models[]`** — per Bedrock target: `source_model_id`, `aws_model_id`, `capabilities_matched[]`, `capability_gaps[]`, `honest_assessment`, `source_provider_price`, `bedrock_price`, `price_comparison`, `migration_complexity`, `model_change` (bool), - `migration_path` (see below), `quota_risk` (`low|medium|high`). Traditional-AI capabilities - do NOT appear here — they are `design_blocks[]` rows with `target_aws_service`. + `migration_path` (see below), `quota_risk` (`low|medium|high`), `required_client_api` + (`responses | chat_completions | converse | invoke | messages` — the client API surface the + generated code must use; for `runtime_openai_cris` it is the API the source app already + calls, never an implicit Converse), and `mantle_selector` (REQUIRED on every `mantle_*` row, + absent otherwise — see below). Traditional-AI capabilities do NOT appear here — they are + `design_blocks[]` rows with `target_aws_service`. - **`capability_mapping`** — per capability that is `true` in the profile's `integration.capabilities_summary`: `{ parity: "full|partial|none", notes }`. - **`code_migration`** — `primary_pattern` (matches profile `integration.pattern`), `framework`, `files_to_modify[]`, `dependency_changes`, and — when a retained source's `source_service` is `azure_openai` or a `provider`/`source_service` pair resolves to OpenAI — `openrouter_path` when a router was detected - (`same_model_mantle|direct|litellm|keep_openrouter`). + (`same_model_runtime|same_model_mantle|direct|litellm|keep_openrouter`; `same_model_runtime` + is the default same-model route — `bedrock-runtime` CRIS — and `same_model_mantle` requires a + `mantle_selector` on the row). - **`infrastructure[]`** — source-cloud AI infra resource → AWS equivalent, with confidence. - **`services_to_migrate[]`** — source-cloud AI service → AWS service, effort, notes. ## migration_path vocabulary -`mantle_openai_responses` (an OpenAI-protocol source — direct or Azure-fronted — with the model -on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `mantle_openai_chat` -(same, preserving a Chat Completions source on a model whose card lists Chat Completions on -mantle — GPT-6, GPT-5.5, GPT-5.4) · `runtime_openai_cris` (same-model move served via -`bedrock-runtime` with a CRIS id, `model_change: false`; `us.`/`in.`/`global.` for GPT-5.6 Terra -and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4, per their model cards) · `converse` (Bedrock-native +**`bedrock-runtime` is the default endpoint for every new path** (`vendored/ai/bedrock-endpoint-policy.md`): +a same-model OpenAI-protocol source records `runtime_openai_cris`, an Anthropic SDK source records +`runtime_messages`, and a `mantle_*` path is recorded only with a `bedrock_models[].mantle_selector`. + +`runtime_openai_cris` (same-model move served via `bedrock-runtime` `/openai/v1` with a CRIS id, +`model_change: false`, `required_client_api` = the source app's `responses` or `chat_completions`; +`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, +GPT-5.5 and GPT-5.4, per their model cards) · `runtime_messages` (Anthropic SDK kept on +`bedrock-runtime` `/anthropic` with a Claude profile id, `required_client_api: "messages"`, +`model_change` per the model mapping) · `mantle_openai_responses` (an OpenAI-protocol source — direct +or Azure-fronted — with the model on Bedrock and the region carrying it: keep the SDK, +`model_change: false`; requires `mantle_selector`) · `mantle_openai_chat` (same, preserving a Chat +Completions source on a model whose card lists Chat Completions on mantle — GPT-6, GPT-5.5, +GPT-5.4; requires `mantle_selector`) · `mantle_messages` (Anthropic SDK on the Mantle `/anthropic` +endpoint; requires `mantle_selector`, normally `workspaces`) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, intelligent prompt routing, or another feature no same-model path serves) · `gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK @@ -203,6 +218,14 @@ swap). A **bare** proprietary `openai.gpt-*` model ID is NEVER paired with a `bedrock-runtime` carries a CRIS-prefixed id (there is no runtime in-region form for GPT-6, GPT-5.6, GPT-5.5 or GPT-5.4). +## mantle_selector (required on every `mantle_*` `bedrock_models[]` row) + +One of `background_inference | server_side_tools | web_search | projects | workspaces | +mantle_only_model_or_api | in_region_processing` — the verified Mantle-only need that justified +leaving the `bedrock-runtime` default (definitions in `vendored/ai/bedrock-endpoint-policy.md`). +API continuity (the app already calls Responses, Chat Completions or Messages) is not a selector. +Absent on `runtime_*`, `converse`, `gpt-oss` and `direct` rows. + ## regional_validation `checked` (`aws___get_regional_availability` on the AWS MCP Server confirmed model/region @@ -267,7 +290,10 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4). - [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves to direct OpenAI: every source model that is available on Bedrock and carried by the - target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. + target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute — + and records `migration_path: "runtime_openai_cris"` by default. +- [ ] Every `bedrock_models[]` row whose `migration_path` starts with `mantle_` carries a + `mantle_selector` from the enum above; no `runtime_*`/`converse` row carries one. - [ ] `regional_warnings` and `multi_model_warnings` are present (`[]` when clean). - [ ] If `agentic_profile.is_agentic == true`: `agentic_design` is present with `migration_approach` matching `preferences.json`; a `harness` source whose origin resolves diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index 3e182e26c..1fb918c0d 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -357,7 +357,7 @@ For each model in `models[]`, record: | GCP Pattern | AWS Pattern | Effort | | ------------ | ------------------------------------------------------------------------- | ------- | -| `direct_sdk` | Mantle Responses API (OpenAI source, model on Bedrock, region carries it) | Minimal | +| `direct_sdk` | Runtime OpenAI API — same model on `bedrock-runtime` CRIS (OpenAI source, model on Bedrock); Mantle only for a recorded `mantle_selector` | Minimal | | `direct_sdk` | Bedrock SDK (boto3 / AWS SDK) | Medium | | `framework` | LangChain/LlamaIndex + Bedrock | Low | | `rest_api` | Bedrock REST API | Medium | @@ -391,9 +391,10 @@ For each detected `integration.pattern` and `ai_source`, generate before/after m | Pattern | Source | Target | Key Change | | ------------------------------- | ------------------------- | ------------------- | ---------------------------------------------------------------------------------------------------- | -| Direct SDK (OpenAI), same model | OpenAI | Selected Mantle API | Base URL → `.../openai/v1`, Bedrock credential, `openai.gpt-*` model ID, `bedrock-mantle:*` IAM | +| Direct SDK (OpenAI), same model | OpenAI | Runtime OpenAI API (default) | Base URL → `https://bedrock-runtime.{region}.amazonaws.com/openai/v1`, Bedrock token, CRIS `us.`/`global.` model ID, `bedrock:InvokeModel*` IAM; same API the source calls | +| Direct SDK (OpenAI), same model | OpenAI | Selected Mantle API (selector-gated) | Only with `mantle_selector`: base URL → `.../openai/v1` on Mantle, Bedrock credential, `openai.gpt-*` model ID, `bedrock-mantle:*` IAM | | Direct SDK | Vertex AI | boto3 Converse API | `generate_content()` → `converse()` | -| Direct SDK | OpenAI | boto3 Converse API | `completions.create()` → `converse()` (use if Mantle region unavailable or Converse features needed) | +| Direct SDK | OpenAI | boto3 Converse API | `completions.create()` → `converse()` (use only for a need no same-model path serves, e.g. Knowledge Bases) | | Direct SDK | Anthropic | boto3 Converse API | `messages.create()` → `converse()` with Claude model ID on Bedrock | | LangChain | ChatVertexAI / ChatOpenAI | ChatBedrock | Swap import and model_id | | LlamaIndex | Vertex / OpenAI LLM | BedrockConverse | Swap import | @@ -402,10 +403,18 @@ For each detected `integration.pattern` and `ai_source`, generate before/after m | Streaming | `stream=True` | `converse_stream` | Event loop over `contentBlockDelta` | **OpenAI model/API selection:** preserve the model and the supported API chosen in the -mapping guide. For Astra on Standard Mantle in `us-east-1` or `us-west-2`, a Chat source selects -`migration_path: "mantle_openai_chat"`; a Responses source selects `"mantle_openai_responses"`. -Keep the OpenAI SDK, use `/openai/v1`, Bedrock credentials and Mantle IAM. Do not reshape -Astra Chat solely because it is proprietary GPT. Record `model_change: false` only when +mapping guide. **The default for every proprietary GPT target is `migration_path: +"runtime_openai_cris"`** with `bedrock_models[].required_client_api` set to the API the source app +calls (`responses` for a Responses source, `chat_completions` for a Chat source) — the OpenAI SDK +stays, the base URL becomes `https://bedrock-runtime.{region}.amazonaws.com/openai/v1`, the model +ID becomes the CRIS profile and IAM is `bedrock:InvokeModel*`. A `mantle_*` path +(`"mantle_openai_chat"` for a Chat source, `"mantle_openai_responses"` for a Responses source — e.g. +Astra on Standard Mantle in `us-east-1` or `us-west-2`) is recorded **only** when the row also +records `bedrock_models[].mantle_selector` from `references/vendored/ai/bedrock-endpoint-policy.md` +(background inference, server-side tools / Web Search, Projects / Workspaces, a Mantle-only model or +API, or strict in-Region processing where runtime offers only CRIS); then keep the OpenAI SDK, use +`/openai/v1` on Mantle, Bedrock credentials and Mantle IAM. API continuity alone never selects +Mantle. Do not reshape Astra Chat solely because it is proprietary GPT. Record `model_change: false` only when the exact source model is retained; a Pro-to-Astra upgrade is a model change requiring eval. GPT-5.x retains its separately dated API evidence and any selected Chat-to-Responses reshape. @@ -423,8 +432,9 @@ GPT-5.5 / GPT-5.4 (model cards read 2026-11-06: `us.openai.gpt-5.5` / `global.op have a `bedrock-runtime` CRIS path (`us.`/`global.` prefixed IDs, per `openai-on-bedrock.md` § Access Paths). No proprietary GPT model has a runtime in-region form. For all of them, a region or feature need (Guardrails, invocation logging, application inference profiles, or a region -outside the mantle matrix) is satisfied by **switching endpoint, not model**: record -`migration_path: "runtime_openai_cris"` with `model_change: false`. Reserve +outside the mantle matrix) is satisfied by **the default endpoint, not a model switch**: record +`migration_path: "runtime_openai_cris"` with `model_change: false` — the same path a region-only +move records. Reserve `migration_path: "converse"` with `model_change: true` for a need the same-model CRIS path itself cannot serve (Knowledge Bases, intelligent prompt routing, or a feature CRIS does not support — see the runtime-side feature splits in `openai-on-bedrock.md`). Note: @@ -435,8 +445,10 @@ must verify against Astra's model card per-feature, not assume parity with Sol/L **When Guardrails is the driving need and the design keeps `runtime_openai_cris` (endpoint switch, not model switch), it MUST also record `bedrock_models[].required_client_api` so the generated client uses a Guardrails-capable surface — `converse` for Astra and 6.1 Terra; `converse` / -`chat_completions` / `invoke` (never `responses`) for 6.1 Sol and the other GPT-6/5.6 CRIS models except Astra and 6.1 Terra. Do not leave the client on `responses` when Guardrails is required.** `Responses` -stays the default for a region-only CRIS move with no Guardrails or application-inference-profile need. +`chat_completions` / `invoke` (never `responses`) for 6.1 Sol and the other GPT-6/5.6 CRIS models except Astra and 6.1 Terra. Do not leave the client on `responses` when Guardrails is required.** For a +region-only CRIS move with no Guardrails or application-inference-profile need, the default +`required_client_api` is the API the source app already calls (`responses` or `chat_completions`) +— never an implicit Converse. **Application inference profiles are Converse-only on GPT-5.5, GPT-5.4, GPT-6.1 Terra and Astra — the requirement MUST reach `required_client_api`.** The GPT-5.4 / GPT-5.5 model cards (read 2026-11-07) @@ -467,6 +479,8 @@ do not manufacture a profile from the region. Keep each unit's model/API/profile For multiple advisor units, write `migration_path` and `required_client_api` on each corresponding `bedrock_models[]` row with its `aws_model_id`; `code_migration` remains the primary-unit default. + + **Throughput:** load `references/vendored/ai/ai-migration-guardrails.md` and apply the selected model/endpoint's quota rules. Astra runtime uses input TPM plus 10 times output TPM. Astra Mantle quotas and cache exemptions must be verified independently; do not copy GPT-5.6's @@ -478,7 +492,7 @@ Generate concrete code examples using actual model IDs from the selected Bedrock **OpenRouter-specific guidance** (if `gateway_type == "llm_router"` AND `detection_signals` contains OpenRouter evidence): -OpenRouter is a hosted routing service (not self-hosted like LiteLLM). It adds a margin on top of provider pricing. **First check whether the underlying model is an OpenAI model with a Mantle target** (`ai_source == "openai"`, per the Mantle guidance above) — OpenRouter is a transport layer, not a different model, so an OpenAI model routed through OpenRouter is eligible for the same same-model Mantle path a direct-SDK OpenAI source gets. Present the options below to the user: +OpenRouter is a hosted routing service (not self-hosted like LiteLLM). It adds a margin on top of provider pricing. **First check whether the underlying model is an OpenAI model that is on Bedrock** (`ai_source == "openai"`, per the OpenAI model/API selection above) — OpenRouter is a transport layer, not a different model, so an OpenAI model routed through OpenRouter is eligible for the same same-model `bedrock-runtime` path a direct-SDK OpenAI source gets. Present the options below to the user: > **If the startup was using OpenRouter primarily for cost-based routing within one model family** > (e.g., routing between Claude Haiku and Claude Sonnet, or Nova Lite and Nova Pro), @@ -488,12 +502,12 @@ OpenRouter is a hosted routing service (not self-hosted like LiteLLM). It adds a | Option | Action | Effort | Trade-off | | -------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| A) Same model via Mantle (recommended when the source model is on Bedrock) | Remove OpenRouter, point the existing OpenAI SDK at Bedrock Mantle with the same model ID | Days | Keeps the same model — no eval/prompt-behavior risk; requires base URL, credential, and model-ID-format changes (e.g. `openai/gpt-4o` → an OpenAI-compatible Bedrock model ID); not a one-line swap | -| B) Direct Bedrock, cross-family | Remove OpenRouter, call Bedrock API directly with a Claude/Nova model | 1–2 weeks | Removes middleman + margin; requires SDK + prompt changes; use when no Mantle target exists or cost is the priority | +| A) Same model on Bedrock (recommended when the source model is on Bedrock) | Remove OpenRouter, point the existing OpenAI SDK at `bedrock-runtime` `/openai/v1` with the same model's CRIS ID (`same_model_runtime`, default); Mantle (`same_model_mantle`) only with a recorded `mantle_selector` | Days | Keeps the same model — no eval/prompt-behavior risk; requires base URL, credential, and model-ID-format changes (e.g. `openai/gpt-4o` → a CRIS-prefixed Bedrock model ID); not a one-line swap | +| B) Direct Bedrock, cross-family | Remove OpenRouter, call Bedrock API directly with a Claude/Nova model | 1–2 weeks | Removes middleman + margin; requires SDK + prompt changes; use when the source model is not on Bedrock or cost is the priority | | C) Self-hosted LiteLLM | Replace OpenRouter with LiteLLM proxy pointing to Bedrock | 1–3 days | Preserves router pattern; removes OpenRouter dependency; adds self-hosting | | D) Keep OpenRouter | Use OpenRouter with `amazon/` prefixed Bedrock models | Hours | Lowest effort; retains OpenRouter dependency and margin; this is a model switch, not a same-model move | -Recommend **A** when `ai_source == "openai"` and a Mantle target exists (region gate passes per the Mantle guidance above); otherwise recommend **B** and present C/D as lower-effort alternatives. Record user's choice (or the recommended default if not asked) in `aws-design-ai.json` → `code_migration.openrouter_path`: `"same_model_mantle"` / `"direct"` / `"litellm"` / `"keep_openrouter"`. +Recommend **A** when `ai_source == "openai"` and the source model is on Bedrock (region gate passes per the mapping guide); A defaults to `same_model_runtime` (Runtime CRIS) and becomes `same_model_mantle` only when the row records a `mantle_selector`. Otherwise recommend **B** and present C/D as lower-effort alternatives. Record user's choice (or the recommended default if not asked) in `aws-design-ai.json` → `code_migration.openrouter_path`: `"same_model_runtime"` / `"same_model_mantle"` / `"direct"` / `"litellm"` / `"keep_openrouter"`. --- @@ -531,6 +545,7 @@ Write `aws-design-ai.json` to `$MIGRATION_DIR/`. - [ ] `honest_assessment` logic is consistent (weakest model drives overall) - [ ] If `ai_source` includes `"openai"`: every model that `shared/openai-on-bedrock.md` lists as available on Bedrock, and whose target region carries it, is mapped to **itself** with `model_change: false` — not to a Claude/Nova substitute - [ ] If `honest_assessment == "recommend_stay"`: `honest_assessment_reason` names a non-cost blocker (cost parity alone is never sufficient for an OpenAI source) +- [ ] Every same-model proprietary GPT row records `migration_path: "runtime_openai_cris"` by default; every `bedrock_models[]` row whose `migration_path` starts with `mantle_` carries a `mantle_selector` from `references/vendored/ai/bedrock-endpoint-policy.md`, and no `runtime_*`/`converse` row carries one - [ ] No `bedrock_models[]` entry pairs a **bare** proprietary GPT id (including bare `openai.gpt-5.5` / `openai.gpt-5.4`) with a Converse/`bedrock-runtime` migration path. GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 MAY use `migration_path: "runtime_openai_cris"` on diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md index e4ab7a770..1500313e5 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md @@ -321,6 +321,8 @@ in this order — the OpenRouter subroute wins over `migration_path`, because 1. **OpenRouter subroute**, when `aws-design-ai.json` → `code_migration.openrouter_path` is set, or `ai-workload-profile.json` has a `base_url` containing `openrouter.ai`: + - `"same_model_runtime"` → `[C] Generate the Bedrock runtime (CRIS) migration script and A/B test harness` + (Step 3B skips gateway config and runs Step 1M's CRIS variant; no provider adapter). - `"same_model_mantle"` → `[C] Generate the Mantle migration script and A/B test harness` (Step 3B skips gateway config and runs Step 1M; no provider adapter). - `"direct"` → `[C] Generate the Bedrock setup script, provider adapter, and A/B test harness` @@ -328,9 +330,11 @@ in this order — the OpenRouter subroute wins over `migration_path`, because - `"litellm"`, `"keep_openrouter"`, or absent → `[C] Generate the gateway config, Bedrock setup script, and A/B test harness` (absent defaults to litellm). Do not promise a provider adapter on these three. 2. **Otherwise** read `ai_architecture.code_migration.migration_path`. Design - records `"mantle_openai_responses"` (not `"mantle"`) and `"converse"`; accept - both Mantle spellings: - - `"mantle"` or `"mantle_openai_responses"` → the Mantle label above. + records `"runtime_openai_cris"` by default, `"mantle_openai_responses"` / + `"mantle_openai_chat"` (not `"mantle"`) only with a `mantle_selector`, and + `"converse"`; accept every Mantle spelling: + - `"runtime_openai_cris"` → the Bedrock runtime (CRIS) label above. + - `"mantle"`, `"mantle_openai_responses"` or `"mantle_openai_chat"` → the Mantle label above. - `"converse"`, `"direct"`, or `"gpt-oss"` → the adapter label above. `"converse"` is a Converse client swap, not a Mantle script and not gateway config. - `"llm_router"`, `"api_gateway"`, `"voice_platform"`, or `"framework"` → the diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md index 5fa5c23ee..137f8b153 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md @@ -28,14 +28,14 @@ Check `preferences.json` → `ai_constraints.ai_framework` to determine the migr | Voice Platform (Vapi, Bland.ai) | Check native Bedrock support, update dashboard | Dashboard config | | Framework (LangChain, LlamaIndex) | Swap provider import (e.g., `ChatBedrock` for `ChatVertexAI`) | 1-5 lines of code | -**OpenAI SDK users via Mantle (config path for a Responses source; small reshape for a Chat Completions source)** — `ai_framework` = `direct` AND `ai_source` = `openai` AND (`migration_path` **starts with** `mantle` (`mantle`, `mantle_openai_responses`, `mantle_openai_chat`) OR `migration_path` == `"runtime_openai_cris"`): +**OpenAI SDK users keeping the same model (config path for a Responses source; small reshape for a Chat Completions source)** — `ai_framework` = `direct` AND `ai_source` = `openai` AND (`migration_path` == `"runtime_openai_cris"` (the default) OR `migration_path` **starts with** `mantle` (`mantle`, `mantle_openai_responses`, `mantle_openai_chat`, selector-gated)): -- `mantle` / `mantle_openai_responses`: point the client at `https://bedrock-mantle.{region}.api.aws/openai/v1` — note the `openai/v1` segment for proprietary GPT models; a bare `/v1` 404s. Swap the credential for a Bedrock API key or the auto-refreshing token provider — **not** the existing OpenAI key. Update the model string to the bare Bedrock model ID (`openai.gpt-*` for a same-model move). Grant the `bedrock-mantle:*` actions; `bedrock:InvokeModel` does not authorize these models. -- `runtime_openai_cris` (GPT-5.6/GPT-6/GPT-5.5/GPT-5.4 via `bedrock-runtime` CRIS): point the client at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` instead — **not** the mantle URL. Update the model string to the CRIS inference-profile ID the selected model's card lists (`us.`/`global.`; `in.` exists for GPT-5.6 Terra and Luna only), not the bare ID. Grant standard `bedrock:InvokeModel*` actions against the CRIS inference-profile ARN — **not** `bedrock-mantle:*`. For a GPT-6 CRIS id, and for the GPT-5.5 / GPT-5.4 CRIS ids (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`, or their `inference-profile/` ARNs), also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` alongside the inference-profile ARN (per the GPT-6 model cards and the GPT-5.5 / GPT-5.4 cards, read 2026-11-07); `iam_policy.py` emits this third ARN for exactly those ids. Check `aws-design-ai.json` → `ai_architecture.bedrock_models[].required_client_api`: if non-Responses, emit that surface (e.g. Converse) instead of defaulting to `responses.create` — absent/`responses` keeps the Responses surface for a region-only move. +- `runtime_openai_cris` (the default — GPT-5.6/GPT-6/GPT-5.5/GPT-5.4 via `bedrock-runtime` CRIS): point the client at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` — **not** the mantle URL. Update the model string to the CRIS inference-profile ID the selected model's card lists (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), not the bare ID. Grant standard `bedrock:InvokeModel*` actions against the CRIS inference-profile ARN — **not** `bedrock-mantle:*`. For a GPT-6 CRIS id, and for the GPT-5.5 / GPT-5.4 CRIS ids (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`, or their `inference-profile/` ARNs), also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` alongside the inference-profile ARN (per the GPT-6 model cards and the GPT-5.5 / GPT-5.4 cards, read 2026-11-07); `iam_policy.py` emits this third ARN for exactly those ids. Read `aws-design-ai.json` → `ai_architecture.bedrock_models[].required_client_api` — the API the source app already calls — and emit that surface (`responses.create`, `chat.completions.create`, or Converse when Design recorded `converse` for a Guardrails or application-inference-profile need); for a region-only move it is simply the source's own API. +- `mantle` / `mantle_openai_responses` / `mantle_openai_chat` (only when Design recorded a `mantle_selector`, see `references/vendored/ai/bedrock-endpoint-policy.md`): point the client at `https://bedrock-mantle.{region}.api.aws/openai/v1` — note the `openai/v1` segment for proprietary GPT models; a bare `/v1` 404s. Swap the credential for a Bedrock API key or the auto-refreshing token provider — **not** the existing OpenAI key. Update the model string to the bare Bedrock model ID (`openai.gpt-*` for a same-model move). Grant the `bedrock-mantle:*` actions; `bedrock:InvokeModel` does not authorize these models. - **Config-only if the source already calls `responses.create`.** A Chat Completions source for GPT-6 or GPT-5.6 may keep `chat.completions.create`/`choices[0].message` parsing unchanged — Chat Completions is supported on both mantle and runtime CRIS for these models, so reshape to Responses is optional (only for a Responses-only feature or demonstrated constraint). GPT-5.5/GPT-5.4 follow the same rule — their model cards (read 2026-11-06) list Chat Completions on both mantle and runtime, so the reshape is optional for them too; probe the selected API against the target account before committing, as for any model. - Test in staging, validate responses -Match the `migration_path` prefix, not the exact string: Design writes the more specific `mantle_openai_responses` for a same-model OpenAI migration, and an equality test against `mantle` would misroute those runs into the adapter path below. `runtime_openai_cris` (GPT-5.6/GPT-6 served via `bedrock-runtime` CRIS instead of `bedrock-mantle`) is matched explicitly alongside the `mantle*` prefix — it does not start with `mantle`, so it needs its own check to avoid falling through to the adapter path below. +Match `runtime_openai_cris` explicitly (it is the default Design writes for a same-model OpenAI migration and does not start with `mantle`, so it needs its own check to avoid falling through to the adapter path below), and match the `mantle` prefix, not the exact string, for the selector-gated Mantle rows: Design writes the more specific `mantle_openai_responses` / `mantle_openai_chat`, and an equality test against `mantle` would misroute those runs into the adapter path below. **Direct SDK users (adapter path — the long pole)** — `ai_framework` = `direct` AND (`ai_source` != `openai` OR `migration_path` = `converse`) AND `migration_path` != `"runtime_openai_cris"`: diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md index 9ef33ae64..d9bdc22f7 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -51,18 +51,22 @@ If any required file is missing: **STOP**. Output: "Missing required artifact: [ Check `preferences.json` → `ai_constraints.ai_framework.value` and `aws-design-ai.json` → `ai_architecture.code_migration.migration_path`: -- `migration_path` starts with `"mantle"` (`"mantle"`, `"mantle_openai_chat"`, `"mantle_openai_responses"`) → Generate Mantle setup (Step 1M) + setup (Step 3) + test harness (Step 2). Skip provider adapter (Step 1). Design records `"mantle_openai_responses"`; treat that spelling as this same Mantle path. -- `migration_path` = `"runtime_openai_cris"` → Generate the CRIS-aware Mantle-equivalent script - (Step 1M, using the `us.`/`global.`-prefixed inference-profile ID and the `bedrock-runtime` - base URL instead of `bedrock-mantle`) + setup (Step 3) + test harness (Step 2). Skip provider - adapter (Step 1) — same as `"mantle"`, just a different endpoint and model-id form. +- `migration_path` = `"runtime_openai_cris"` (**the default same-model path**; Design records it + for every proprietary GPT target unless a `mantle_selector` is present) → Generate the CRIS + variant of the same-model script (Step 1M, using the `us.`/`global.`-prefixed inference-profile + ID, the `bedrock-runtime` base URL `https://bedrock-runtime.{region}.amazonaws.com/openai/v1`, + `bedrock:InvokeModel*` IAM and the row's `required_client_api`) + setup (Step 3) + test harness + (Step 2). Skip provider adapter (Step 1). +- `migration_path` starts with `"mantle"` (`"mantle"`, `"mantle_openai_chat"`, `"mantle_openai_responses"`) → Generate Mantle setup (Step 1M) + setup (Step 3) + test harness (Step 2). Skip provider adapter (Step 1). Design records `"mantle_openai_responses"` or `"mantle_openai_chat"` only together with `bedrock_models[].mantle_selector` (see `references/vendored/ai/bedrock-endpoint-policy.md`); echo that selector in the generated script's header so the Mantle choice stays traceable. Treat the `"mantle_openai_responses"` spelling as this same Mantle path. - `migration_path` = `"gpt-oss"` → Generate provider adapter (Step 1) targeting the gpt-oss model on Bedrock via the Converse API + setup (Step 3) + test harness (Step 2). Use the Bedrock model ID from `aws-design-ai.json` → `ai_architecture.bedrock_models[].aws_model_id` (the gpt-oss model ID). Do NOT generate a Mantle script — gpt-oss uses the Converse API directly. - `"converse"`, `"direct"`, or absent → Generate provider adapter (Step 1) + setup (Step 3) + test harness (Step 2). `"converse"` is a Converse client swap (Design records it when a proprietary model cannot stay on Mantle); do NOT generate a Mantle script for it. -- `"llm_router"`, `"api_gateway"`, `"voice_platform"`, or `"framework"` → Skip Step 1, generate gateway config (Step 3B) instead. Step 3B's OpenRouter subroute can still replace that gateway config with Step 1M (`same_model_mantle`) or the provider adapter (`direct`); do not decide those two from `migration_path` alone. +- `"llm_router"`, `"api_gateway"`, `"voice_platform"`, or `"framework"` → Skip Step 1, generate gateway config (Step 3B) instead. Step 3B's OpenRouter subroute can still replace that gateway config with Step 1M (`same_model_runtime` → the CRIS variant, `same_model_mantle` → the Mantle variant) or the provider adapter (`direct`); do not decide those from `migration_path` alone. -For `runtime_openai_cris`, read the matched `bedrock_models[].required_client_api`. -Use `responses` for current Design plans without that field; only legacy plans fall back -to the row or `code_migration.runtime_api` and their Converse default. Use the resolved +For `runtime_openai_cris`, read the matched `bedrock_models[].required_client_api` — the API the +source app already calls (`responses` or `chat_completions`; `converse` only when Design recorded +it for a Converse-only feature). A current Design plan always records it; a legacy plan without +the field falls back to the row or `code_migration.runtime_api`, and if neither is present the +source API must be resolved before generating — never assume Converse. Use the resolved API consistently for the generated client, comparison and setup calls. Preserve the exact `aws_model_id` string, whether a short profile ID or a system-profile ARN. When building the IAM policy, use a supplied profile ARN directly and resolve its backing @@ -153,7 +157,7 @@ Generate `ai-migration/test_comparison.py` — always Python regardless of adapt - Output structured JSON to `--output` (default: `comparison_results.json`) - Built-in test prompts: include 3-5 prompts based on `ai-workload-profile.json` → `models[].usage_context` covering the primary use case - **Adapter paths** (`direct`, `converse`, `gpt-oss`, or a gateway subroute that emits an adapter): use `from provider_adapter import get_provider` from the emitted provider adapter. -- **Mantle / runtime CRIS / `same_model_mantle` paths:** do not import `provider_adapter` because Step 0 intentionally skips it. Call the source using its recorded SDK and call the target directly with the selected Mantle API. Use this target function for proprietary GPT plans, substituting the saved design values: +- **Mantle / runtime CRIS / `same_model_runtime` / `same_model_mantle` paths:** do not import `provider_adapter` because Step 0 intentionally skips it. Call the source using its recorded SDK and call the target directly with the selected Mantle API. Use this target function for proprietary GPT plans, substituting the saved design values: ```python def call_bedrock(prompt): @@ -234,7 +238,8 @@ Skip if `ai_framework` = `"direct"` or absent. Read `preferences.json` → `ai_c Check `aws-design-ai.json` → `code_migration.openrouter_path` (set in Design Part 5): -- `"same_model_mantle"` → Skip gateway config. Run **Step 1M** (Mantle migration script) using the OpenRouter-sourced model ID as the "current model ID" input. Add a header note that Step 1M's usual "no code changes required" claim does NOT hold here: OpenRouter → Mantle changes the base URL, credential type (Bedrock API key/token provider, not an OpenRouter key), and the model-ID format (e.g. `openai/gpt-4o` → the Mantle model ID). Print this as an explicit line in the generated script rather than reusing Step 1M's unchanged-code language verbatim. +- `"same_model_runtime"` (the default same-model subroute) → Skip gateway config. Run **Step 1M** in its CRIS variant (`bedrock-runtime` base URL `https://bedrock-runtime.{region}.amazonaws.com/openai/v1`, the CRIS inference-profile ID, `bedrock:InvokeModel*` IAM, the row's `required_client_api`) using the OpenRouter-sourced model ID as the "current model ID" input. Add a header note that Step 1M's usual "no code changes required" claim does NOT hold here: OpenRouter → `bedrock-runtime` changes the base URL, credential type (Bedrock token/API key, not an OpenRouter key), and the model-ID format (e.g. `openai/gpt-4o` → the CRIS-prefixed Bedrock model ID). Print this as an explicit line in the generated script rather than reusing Step 1M's unchanged-code language verbatim. +- `"same_model_mantle"` (only when the row carries a `mantle_selector`) → Skip gateway config. Run **Step 1M** (Mantle migration script) using the OpenRouter-sourced model ID as the "current model ID" input. Add the same header note: OpenRouter → Mantle changes the base URL, credential type (Bedrock API key/token provider, not an OpenRouter key), and the model-ID format (e.g. `openai/gpt-4o` → the Mantle model ID). Print this as an explicit line in the generated script rather than reusing Step 1M's unchanged-code language verbatim. - `"direct"` → Skip gateway config. Generate provider adapter instead (same as Step 1 for direct SDK users). Note: "Migrating from OpenRouter to direct Bedrock API removes the routing middleman and OpenRouter's margin." - `"litellm"` → Generate `gateway_config.yaml` in LiteLLM format (standard `llm_router` logic above) + add header comment: "# Migration from OpenRouter to self-hosted LiteLLM with Bedrock backend\n# Install: pip install litellm\n# Run: litellm --config gateway_config.yaml" - `"keep_openrouter"` → Generate `gateway_config.json` with OpenRouter model ID mappings: diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md index b669556ad..d986ee273 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md @@ -178,7 +178,32 @@ path for any GPT-6 model. ### Client setup -Requires the OpenAI SDK at **>= 2.45.0**. Preferred client auto-refreshes a short-term Bedrock token: +Requires the OpenAI SDK at **>= 2.45.0**. The default client targets `bedrock-runtime` (`runtime_openai_cris`) and +auto-refreshes a short-term Bedrock token through a callable `api_key`: + +```python +from aws_bedrock_token_generator import provide_token +from openai import OpenAI + +region = "us-east-1" +client = OpenAI( + base_url=f"https://bedrock-runtime.{region}.amazonaws.com/openai/v1", + api_key=lambda: provide_token(region=region), + max_retries=6, +) + +response = client.responses.create( + model="us.openai.gpt-5.6-terra", # the CRIS inference-profile id, never the bare id, on runtime + input="...", + reasoning={"effort": "medium"}, +) +``` + +Keep the API the source app already calls (`responses.create` or `chat.completions.create`); the runtime endpoint +serves both for these models. + +**Mantle form (only when Design recorded a `mantle_selector`):** the `BedrockOpenAI` client targets +`bedrock-mantle` with the bare model id: ```python from aws_bedrock_token_generator import provide_token @@ -198,7 +223,7 @@ response = client.responses.create( ) ``` -The alternative — `OpenAI(base_url=".../openai/v1", api_key=os.environ["AWS_BEARER_TOKEN_BEDROCK"])` — uses a key +The alternative on either endpoint — `api_key=os.environ["AWS_BEARER_TOKEN_BEDROCK"]` — uses a key that expires within 12 hours and is not refreshed. Do not recommend it for production. **IAM:** on the mantle path, the managed policy `AmazonBedrockMantleInferenceAccess` grants what inference needs, diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md index a546f0323..a64fc8b51 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md @@ -78,9 +78,34 @@ Bedrock-model capabilities: `text_generation`, `structured_output`, `image_gener Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction`, `image_analysis`, `speech_transcription`. +## migration_path, required_client_api, mantle_selector, openrouter_path vocabulary + +`ai_architecture.code_migration.migration_path` and the per-row `bedrock_models[]` fields use the +same vocabulary as the shared contract (`references/vendored/ai/schema-design-aws-ai.md`), with +**`bedrock-runtime` as the default endpoint** (`references/vendored/ai/bedrock-endpoint-policy.md`): + +- `runtime_openai_cris` — same-model OpenAI-protocol move on `bedrock-runtime` `/openai/v1` with a + CRIS id (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, + GPT-6, GPT-5.5 and GPT-5.4), `model_change: false`, `required_client_api` = the API the source + app already calls (`responses` or `chat_completions`; `converse` only for a Converse-only + feature such as Guardrails). This is the default for every proprietary GPT target. +- `runtime_messages` — Anthropic SDK kept on `bedrock-runtime` `/anthropic` with a Claude profile + id, `required_client_api: "messages"`. +- `mantle_openai_responses` / `mantle_openai_chat` / `mantle_messages` — the `bedrock-mantle` + endpoint; each such row MUST carry `mantle_selector`, one of `background_inference | + server_side_tools | web_search | projects | workspaces | mantle_only_model_or_api | + in_region_processing`. API continuity alone is not a selector. +- `converse`, `gpt-oss`, `direct` — Bedrock-native paths; never paired with a **bare** + proprietary `openai.gpt-*` id. +- `code_migration.openrouter_path` (router detected) — `same_model_runtime` (default same-model + route) | `same_model_mantle` (requires `mantle_selector`) | `direct` | `litellm` | + `keep_openrouter`. + ## Validation Checklist - [ ] `metadata.ai_source` matches `summary.ai_source` from `ai-workload-profile.json`. +- [ ] Every `bedrock_models[]` row whose `migration_path` starts with `mantle_` carries a + `mantle_selector`; a same-model proprietary GPT row records `runtime_openai_cris` by default. - [ ] Every `design_blocks[]` row has exactly one of `target_bedrock_model` / `target_aws_service` non-null (XOR). - [ ] Every traditional-AI row (`document_extraction`/`image_analysis`/`speech_transcription`) diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md index 704879840..72770f774 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/README.md @@ -30,6 +30,7 @@ if this skill needs one, add the copy **and** a row in the table below. | `ai/ai-openai-to-bedrock.md` | `skills/shared/ai/ai-openai-to-bedrock.md` | | `ai/ai-gemini-to-bedrock.md` | `skills/shared/ai/ai-gemini-to-bedrock.md` | | `ai/ai-openrouter-to-bedrock.md` | `skills/shared/ai/ai-openrouter-to-bedrock.md` | +| `ai/bedrock-endpoint-policy.md` | `skills/shared/ai/bedrock-endpoint-policy.md` | | `ai/bedrock-pricing-cache.md` | `skills/shared/ai/bedrock-pricing-cache.md` | | `ai/bedrock-quotas.md` | `skills/shared/ai/bedrock-quotas.md` | | `ai/design-ref-agentic-to-agentcore.md` | `skills/shared/ai/design-ref-agentic-to-agentcore.md` | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index dba311abf..852c8e2fb 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -83,11 +83,19 @@ GPT-5.4, the target is **the same model on Bedrock**. | GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` / `us.` `global.` prefixed | `strong_migrate` — same model; ~10% over OpenAI std In-Region only (no Geo/Global price published) | Then apply the **region gate** below. Record `model_change: false` and the chosen path in -`aws-design-ai.json` → `ai_architecture.code_migration`: `migration_path: "mantle_openai_chat"` for -sources using Chat Completions on a model whose card lists it on mantle (GPT-6, GPT-5.5, GPT-5.4), or -`"mantle_openai_responses"` for a selected Mantle Responses path. Use `migration_path: "runtime_openai_cris"` -when a GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 target is served via `bedrock-runtime` with a CRIS id (the model -cards recommend runtime for new applications). +`aws-design-ai.json` → `ai_architecture.code_migration`. **The default for every Tier 0 target is +`migration_path: "runtime_openai_cris"`** — the same model served via `bedrock-runtime` with a CRIS +id (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, +GPT-5.5 and GPT-5.4) — with +`bedrock_models[].required_client_api` set to the API the source app already calls (`responses` or +`chat_completions`; `converse` only when a Converse-only feature such as Guardrails is required). +Amazon Bedrock recommends `bedrock-runtime` for new applications +(`references/vendored/ai/bedrock-endpoint-policy.md`). Record `migration_path: "mantle_openai_chat"` +(Chat Completions source on a model whose card lists it on mantle — GPT-6, GPT-5.5, GPT-5.4) or +`"mantle_openai_responses"` **only** when the row also records a `bedrock_models[].mantle_selector` +from that policy file (background inference, server-side tools / Web Search, Projects / Workspaces, +a Mantle-only model or API, or strict in-Region processing where runtime offers only CRIS). API +continuity alone never selects Mantle. Report the assessment honestly: `strong_migrate` here means "low-risk, well-supported move", not "cheaper". @@ -102,26 +110,30 @@ If neither path meets the constraints, offer a supported region or a model chang Do not assume GPT-5.6's broader regional footprint applies to Astra. **GPT-6 Astra / Sol / Luna / 6.1 Sol / 6.1 Terra:** same shape as GPT-5.6 — if the target region is in the mantle in-region matrix (narrower than GPT-5.6's: us-east-1 + us-west-2 for Astra only, us-east-1 only for Sol/Luna/6.1 Sol/6.1 Terra — see -`openai-on-bedrock.md` § Regional Availability), either endpoint works; otherwise the `bedrock-runtime` CRIS path -(`us.`/`global.` prefixed) covers broad commercial regions. GPT-6 Astra's CRIS path additionally lacks `Invoke` +`openai-on-bedrock.md` § Regional Availability), both endpoints work and the `bedrock-runtime` CRIS path +(`us.`/`global.` prefixed) is chosen unless a `mantle_selector` applies; outside that matrix the same CRIS path +covers broad commercial regions. GPT-6 Astra's CRIS path additionally lacks `Invoke` support (Responses/Chat Completions/Converse only) — verify per-feature before assuming parity with Sol/Luna/6.1 Sol. -**GPT-5.6 Sol / Terra / Luna:** rarely blocked. If the target region is in the mantle in-region matrix, either -endpoint works. If not, the `bedrock-runtime` CRIS path (Geo `us.`/`in.`, Global `global.`) covers most commercial -regions — offer it, stating the data-residency implication of cross-region routing (Geo stays within the geography; -Global routes anywhere and is the cost-parity option). Only a workload that requires strictly in-region processing -in a region outside the mantle matrix falls through to Tier 1. +**GPT-5.6 Sol / Terra / Luna:** rarely blocked. The `bedrock-runtime` CRIS path (Geo `us.` — plus `in.` for Terra +and Luna only — and Global `global.`) covers most commercial regions and is the default whether or not the target region is also in the +mantle in-region matrix — state the data-residency implication of cross-region routing (Geo stays within the +geography; Global routes anywhere and is the cost-parity option). A workload that requires strictly in-region +processing in a region inside the mantle matrix records `mantle_selector: "in_region_processing"` and takes the +Mantle path; one that requires it in a region outside the mantle matrix falls through to Tier 1. **GPT-5.5 / GPT-5.4:** same shape as GPT-5.6, with a narrower mantle matrix (GPT-5.5: us-east-1, us-east-2; -GPT-5.4: us-east-1, us-east-2, us-west-2, us-gov-west-1 — per their model cards, read 2026-11-06). If the target -region is in that matrix, either endpoint works. Otherwise the `bedrock-runtime` CRIS path (`us.openai.gpt-5.5` / -`global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`) reaches it, with the same data-residency -caveat as GPT-5.6 (Geo stays within the geography, Global routes anywhere). There is no runtime in-region form. -Only a workload that requires strictly in-region processing in a region outside the mantle matrix falls through: - -1. Prefer offering a **region change** to a supported mantle region, if no data-residency constraint pins the - workload. This preserves the zero-behavior-delta benefit, which is the whole point of Tier 0. +GPT-5.4: us-east-1, us-east-2, us-west-2, us-gov-west-1 — per their model cards, read 2026-11-06). The +`bedrock-runtime` CRIS path (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / +`global.openai.gpt-5.4`) is the default, with the same data-residency caveat as GPT-5.6 (Geo stays within the +geography, Global routes anywhere). There is no runtime in-region form. Strict in-region processing inside the +mantle matrix is a valid `in_region_processing` selector for the Mantle path. Only a workload that requires +strictly in-region processing in a region outside the mantle matrix falls through: + +1. Prefer offering a **region change** to a supported mantle region (recorded with + `mantle_selector: "in_region_processing"`), if no data-residency constraint pins the workload. This preserves + the zero-behavior-delta benefit, which is the whole point of Tier 0. 2. Otherwise fall through to Tier 1 and say why explicitly — a region constraint, not a model judgment. Record the outcome in `regional_warnings[]`. @@ -290,11 +302,11 @@ rather than dismissing it. For a long-context (>272K) workload the gap is much w | OpenAI Feature | Bedrock Equivalent | Notes | | ------------------------------------------ | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | -| OpenAI SDK (direct) | Same model on mantle Responses API | Base URL + credential + model ID change; see `shared/openai-on-bedrock.md` for the `/openai/v1` path | -| Responses API | Mantle Responses API | Closest to a drop-in; still verify the path and model ID | +| OpenAI SDK (direct) | Same model on `bedrock-runtime` `/openai/v1` (CRIS id) | Base URL + credential + model ID change; see `shared/openai-on-bedrock.md` for the `/openai/v1` path; Mantle only with a `mantle_selector` | +| Responses API | Runtime Responses API (`/openai/v1/responses`) | Closest to a drop-in; always synchronous on runtime (`background=true` needs a Mantle selector) | | Chat Completions | Preserve supported Chat when selected | Reshape only for a selected API change; verify the model and endpoint | -| Function calling | Supported on GPT-5.x via mantle; Claude tools via Converse | Same-model keeps tool semantics identical | -| Reasoning effort | `reasoning={"effort": ...}` on mantle | `none` / `low` / `medium` / `high` / `xhigh` / `max` | +| Function calling | Supported on GPT-5.x via runtime or mantle; Claude tools via Converse | Same-model keeps tool semantics identical; server-side tools are Mantle-only | +| Reasoning effort | `reasoning={"effort": ...}` on either endpoint | `none` / `low` / `medium` / `high` / `xhigh` / `max` | | Prompt caching | GPT-5.6 (90% off cached input); GPT-5.5 / 5.4 implicit caching on `bedrock-runtime` Responses only | GPT-5.5 / 5.4 cards (read 2026-11-06) list a cache-read rate and implicit caching on runtime, none on mantle; Claude has its own caching | | Streaming | Supported | Verify per surface | | Vision | GPT-5.x (image input) or Claude / Llama 4 | Same-model preserves behavior | @@ -306,22 +318,34 @@ rather than dismissing it. For a long-context (>272K) workload the gap is much w | Assistants API | See decision tree below | Path depends on which features are used | | Realtime API | No equivalent | Stay on OpenAI for this | | Codex | Not verified on Bedrock — see `shared/openai-on-bedrock.md` | Do not price; re-check the OpenAI model card index first | -| Guardrails / KB / invocation logging | Bedrock-native model or gpt-oss | Not available through the mantle GPT path | +| Guardrails / KB / invocation logging | `bedrock-runtime` CRIS path (Guardrails via Converse; logging on every runtime API); KB needs a Bedrock-native model or gpt-oss | None of these are available through the mantle GPT path | --- ## Common Migration Paths -### Same model on Bedrock (Tier 0) — smallest possible change +### Same model on Bedrock (Tier 0) — `bedrock-runtime` by default -The application keeps the OpenAI SDK and the same model. Changes are limited to: +The application keeps the OpenAI SDK, the same model and the same API it calls today +(`migration_path: "runtime_openai_cris"`, `required_client_api` = `responses` or `chat_completions`). Changes are +limited to: -1. Base URL → `https://bedrock-mantle.{region}.api.aws/openai/v1` (note the `openai/v1` segment) -2. Credentials → Bedrock API key or the auto-refreshing `BedrockOpenAI` token provider, **not** an OpenAI key -3. Model ID → the `openai.gpt-*` form -4. IAM → `bedrock-mantle:*` actions, e.g. via `AmazonBedrockMantleInferenceAccess` +1. Base URL → `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` (note the `openai/v1` segment) +2. Credentials → a short-term Bedrock bearer token (`api_key=lambda: provide_token(region=region)` from + `aws-bedrock-token-generator`) or a Bedrock API key, **not** an OpenAI key +3. Model ID → the CRIS inference-profile id (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only + for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), never the bare `openai.gpt-*` form +4. IAM → `bedrock:InvokeModel*` on the inference-profile and foundation-model ARNs (GPT-6 / GPT-5.5 / GPT-5.4 + also need `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account-id}:project/default`), plus + `bedrock:GetInferenceProfile` on the profile -Full snippet and prerequisites in `shared/openai-on-bedrock.md`. No prompt changes, no eval regression expected — +**Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` +(`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, +the auto-refreshing `BedrockOpenAI` token provider or a Bedrock API key, the bare `openai.gpt-*` id, and +`bedrock-mantle:*` IAM (e.g. `AmazonBedrockMantleInferenceAccess`). Record `migration_path: "mantle_openai_responses"` +or `"mantle_openai_chat"` with the selector. + +Full snippets and prerequisites in `shared/openai-on-bedrock.md`. No prompt changes, no eval regression expected — but still run the eval harness to confirm, because infrastructure and tokenizer-adjacent behavior can differ. ### Assistants API → migration decision tree @@ -329,16 +353,17 @@ but still run the eval harness to confirm, because infrastructure and tokenizer- Assistants API and Responses API are different surfaces. Do not treat all Assistants usage as a config-only change. **1. App already uses the OpenAI Responses API** (`responses.create`) -→ Cleanest path. Base URL + credential + model ID change. +→ Cleanest path. Base URL + credential + model ID change on `bedrock-runtime` (`runtime_openai_cris`). **2. App uses Assistants API only for stateful multi-turn conversation** (no hosted tools, no file search, no code interpreter, no persistent Assistant objects, no complex run lifecycle) -→ Mantle Responses is viable. Requires migrating `threads`/`runs` to `responses.create` — a small API migration -(days), not a redesign. Not a zero-code change. +→ Runtime Responses is viable (stateful `previous_response_id` works on `bedrock-runtime`). Requires migrating +`threads`/`runs` to `responses.create` — a small API migration (days), not a redesign. Not a zero-code change. **3. App uses Assistants API with simple hosted tools** (function calling only) -→ Mantle Responses with tool use is viable. Moderate migration (1–2 weeks) to adapt tool definitions and the run -lifecycle. +→ Runtime Responses with client-side tool use is viable. Moderate migration (1–2 weeks) to adapt tool definitions +and the run lifecycle. Server-side or pre-configured tools (including Web Search) are a `server_side_tools` / +`web_search` Mantle selector. **4. App uses Assistants API with file search, vector stores, code interpreter, persistent Assistant objects, or complex run lifecycle management** @@ -347,15 +372,16 @@ Gateway, gateway-fronted knowledge bases) for a full agentic replacement (2–4 if the team prefers to own state. Never target classic Bedrock Agents (`bedrock-agent`) — it is in maintenance mode and closed to new customers as of July 30, 2026. -### When to prefer a Bedrock-native model over the mantle GPT path +### When to prefer a Bedrock-native model over the same-model GPT path -Choose Claude / Nova via Converse when the workload needs Knowledge Bases, intelligent prompt routing, structured -outputs, or application inference profiles — none of which the mantle GPT path offers. Note what no longer forces -a family switch for GPT-5.6, GPT-5.5 and GPT-5.4: their `bedrock-runtime` CRIS paths provide cross-region -inference, invocation logs, Guardrails and application inference profiles (both Converse API only), and -structured outputs and implicit prompt caching (both Responses API only) — per the GPT-5.4 / GPT-5.5 model cards -read 2026-11-06. Knowledge Bases and intelligent prompt routing still require a Bedrock-native model. The mantle -GPT path is the smallest change; the Bedrock-native path remains the most feature-complete. +Choose Claude / Nova via Converse when the workload needs Knowledge Bases or intelligent prompt routing — neither +is served by a same-model GPT path on either endpoint. Note what no longer forces a family switch for GPT-5.6, +GPT-5.5 and GPT-5.4: their `bedrock-runtime` CRIS paths (the default) provide cross-region inference, invocation +logs, Guardrails and application inference profiles (both Converse API only), and structured outputs and implicit +prompt caching (both Responses API only) — per the GPT-5.4 / GPT-5.5 model cards read 2026-11-06. Knowledge +Bases and intelligent prompt routing still require a Bedrock-native model. The same-model `bedrock-runtime` path +is the default; the Bedrock-native path is the most feature-complete; the mantle GPT path is reserved for a +recorded `mantle_selector`. ### High-spend tiering diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openrouter-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openrouter-to-bedrock.md index ca91e4173..18ff0e281 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openrouter-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openrouter-to-bedrock.md @@ -62,7 +62,7 @@ For every distinct upstream model behind the gateway: 1. **Select the provider guide** from the provider segment (`anthropic` / `openai` / `google`). 2. **Apply that guide's selection policy** to the exact model segment — same-model-on-Bedrock where it exists (OpenAI's Tier 0, Claude version-to-version), else the guide's cross-family / tier mapping. The provider guides own all pricing tables, region gates, and lifecycle checks; do not restate them here. -3. **Record the mapping per upstream, including the guide's selected endpoint** — the gateway's output is a set of `{ upstream_model → bedrock_target }` decisions, not one decision, and each decision carries the `migration_path`, endpoint/inference-profile ID, and credential/IAM requirement the provider guide chose. For GPT targets `ai-openai-to-bedrock.md` may select the Mantle Responses path (`mantle_openai_responses`) **or** the `bedrock-runtime` CRIS path (`runtime_openai_cris`, with a `us.`/`global.` profile) depending on the model and region gate — carry whichever it selected through to cutover, do not collapse every GPT target onto Mantle. Two Claude models behind the same gateway produce two rows, never one. +3. **Record the mapping per upstream, including the guide's selected endpoint** — the gateway's output is a set of `{ upstream_model → bedrock_target }` decisions, not one decision, and each decision carries the `migration_path`, endpoint/inference-profile ID, and credential/IAM requirement the provider guide chose. For GPT targets `ai-openai-to-bedrock.md` selects the `bedrock-runtime` CRIS path by default (`runtime_openai_cris`, with a `us.`/`global.` profile and the source app's `required_client_api`) and the Mantle path (`mantle_openai_responses` / `mantle_openai_chat`) only with a recorded `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`) — carry whichever it selected through to cutover, do not collapse every GPT target onto Mantle. Two Claude models behind the same gateway produce two rows, never one. Preserve the distinction between a provider reached **through OpenRouter** and the **same provider reached directly**. A workload may use direct Anthropic SDK calls *and* Anthropic-via-OpenRouter at once; both map to Bedrock-hosted Claude, but they are different source integrations with different cutover work (a direct SDK swap vs. removing a gateway hop). @@ -72,8 +72,8 @@ Preserve the distinction between a provider reached **through OpenRouter** and t Replacing OpenRouter with Bedrock is an integration change, not just a model change: -- **Transport — per the selected `migration_path`, not one default.** For Claude / Nova / Llama / Gemini-replacement targets, the OpenRouter client becomes the Bedrock Converse API. For GPT targets the transport follows the provider guide's selection: the Bedrock Mantle OpenAI-compatible endpoint when it chose `mantle_openai_responses`, or `bedrock-runtime` with the selected CRIS inference profile when it chose `runtime_openai_cris`. Do not force every GPT target onto Mantle — a runtime/CRIS selection sent to Mantle (or a Mantle-only model routed to runtime) is an incompatible cutover plan. A workload that used OpenRouter precisely for its one-API-many-models convenience may want a thin internal router over Bedrock to preserve that shape — see `references/vendored/ai/ai-migration-guardrails.md`. -- **Credentials.** Remove the OpenRouter key and any BYOK upstream keys; use IAM matched to each target's selected path — `bedrock:InvokeModel` for Converse and the `runtime_openai_cris` path, `bedrock-mantle:*` (e.g. `AmazonBedrockMantleInferenceAccess`) for the Mantle GPT path. The source-key retirement rules apply — do not plant the old gateway/provider keys in the target. +- **Transport — per the selected `migration_path`; the default is `bedrock-runtime`.** For Claude / Nova / Llama / Gemini-replacement targets, the OpenRouter client becomes the Bedrock Converse API. For GPT targets the transport follows the provider guide's selection: `bedrock-runtime` `/openai/v1` with the selected CRIS inference profile when it chose `runtime_openai_cris` (the default — recorded as `openrouter_path: "same_model_runtime"`), or the Bedrock Mantle OpenAI-compatible endpoint only when it chose `mantle_openai_responses` / `mantle_openai_chat` with a `mantle_selector` (`openrouter_path: "same_model_mantle"`). Do not force every GPT target onto Mantle — a runtime/CRIS selection sent to Mantle (or a Mantle-only model routed to runtime) is an incompatible cutover plan. A workload that used OpenRouter precisely for its one-API-many-models convenience may want a thin internal router over Bedrock to preserve that shape — see `references/vendored/ai/ai-migration-guardrails.md`. +- **Credentials.** Remove the OpenRouter key and any BYOK upstream keys; use IAM matched to each target's selected path — `bedrock:InvokeModel*` for Converse and the `runtime_openai_cris` path (the default), `bedrock-mantle:*` (e.g. `AmazonBedrockMantleInferenceAccess`) only for a selector-gated Mantle GPT path. The source-key retirement rules apply — do not plant the old gateway/provider keys in the target. - **Routing policy.** If the app used OpenRouter's automatic model selection or fallback routing, that policy has no Bedrock equivalent and must be reimplemented in the application or dropped deliberately — flag it rather than assuming parity. - **Per-upstream validation.** Validate each mapped target independently (behavior, tool-calling, structured output) the way its provider guide prescribes; a gateway migration is only complete when every retained upstream passes. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md new file mode 100644 index 000000000..fdc4948a2 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md @@ -0,0 +1,115 @@ +# Bedrock Endpoint Policy — `bedrock-runtime` by default, `bedrock-mantle` by selector + +> Canonical endpoint-selection policy for every Bedrock migration path in this plugin. +> Vendored into each consuming skill as `references/vendored/ai/bedrock-endpoint-policy.md` +> and kept byte-identical by `tools/sync-vendored.py`; edit HERE, then run the sync. +> Every statement below comes from the public Amazon Bedrock User Guide pages listed in +> § Sources (read 2026-11-07). Re-read those pages before changing a row. + +## The rule + +**Use `bedrock-runtime` for new applications.** Amazon Bedrock documents `bedrock-runtime` as the +recommended endpoint for most new applications; it serves InvokeModel, Converse, the +OpenAI-compatible Responses and Chat Completions APIs, and the Anthropic Messages API. + +**Use `bedrock-mantle` only for a verified Mantle-only need.** The documented reasons are: + +- background (asynchronous, long-running) inference — Responses `background=true` is rejected + with a 400 on `bedrock-runtime`; +- server-side tool use and pre-configured tools, including Web Search; +- customer-created Projects (OpenAI-compatible) or Workspaces (Anthropic-compatible) — + `bedrock-runtime` exposes only the account's default project; +- a model or API that the Endpoint availability page lists on `bedrock-mantle` only; +- strict in-Region processing for a model that `bedrock-runtime` serves only through a + cross-Region inference (CRIS) profile while `bedrock-mantle` serves it in the required Region. + +**API continuity alone never selects Mantle.** An OpenAI Responses, OpenAI Chat Completions or +Anthropic Messages source keeps its SDK on `bedrock-runtime` by changing the base URL, the +credential and the model ID. Choosing Mantle because "the app already speaks Responses" is not a +selector. + +**Pricing never selects an endpoint.** Per-token pricing for the same model is identical on both +endpoints; choose by API and capability need. + +## `mantle_selector` — the recorded reason + +Every `bedrock_models[]` row whose `migration_path` starts with `mantle_` (`mantle_openai_responses`, +`mantle_openai_chat`, `mantle_messages`) MUST record `mantle_selector` with exactly one value: + +| `mantle_selector` | Verified need | +| --------------------------- | ----------------------------------------------------------------------------------------------- | +| `background_inference` | Responses requests with `background=true` / long-running asynchronous inference | +| `server_side_tools` | Server-side tool use (Lambda-backed MCP tools) or pre-configured tools other than Web Search | +| `web_search` | The built-in Web Search tool | +| `projects` | Customer-created Projects (OpenAI-compatible) for isolation and cost tracking | +| `workspaces` | Workspaces (Anthropic-compatible) referenced by `anthropic-workspace-id` | +| `mantle_only_model_or_api` | The model, or the API for that model, is listed on `bedrock-mantle` only | +| `in_region_processing` | Strict in-Region processing where `bedrock-runtime` offers only a CRIS profile for that model | + +A `mantle_*` row without a `mantle_selector`, or with a reason outside this table, is a design +defect: rewrite the row as the `bedrock-runtime` path (`runtime_openai_cris` with the source +app's `required_client_api`, or `runtime_messages`) or justify it with one of the values above. +A `runtime_*` or `converse` row never carries `mantle_selector`. + +## Public comparison (read 2026-11-07) + +| Item | `bedrock-runtime` | `bedrock-mantle` | Source page | +| ---------------------------------------------- | --------------------------------------------------------------------------------- | --------------------------------------------------------------- | ----------------------------------------------- | +| Recommended default | Yes — "for most new applications" | Only for capabilities available only there | `endpoints.html`, `models-endpoint-availability.html` | +| InvokeModel / Converse | Supported | Not supported | `endpoints.html` | +| OpenAI Responses API | Supported at `/openai/v1`; always synchronous; default project only | Supported at `/v1` (some models `/openai/v1`); background OK | `inference-responses-api.html` | +| OpenAI Chat Completions API | Supported at `/openai/v1/chat/completions` | Supported at `/v1/chat/completions` (some models `/openai/v1`) | `inference-chat-completions.html` | +| Anthropic Messages API | Supported at `/anthropic` (SDK sends `/anthropic/v1/messages`) | Supported at `/anthropic/v1/messages` | `inference-messages-api.html` | +| Background Responses (`background=true`) | Rejected (400) | Supported | `endpoints.html`, `inference-responses-api.html` | +| Server-side tool use / pre-configured tools | Not available; client-side tools work | Supported | `tool-use-server-side.html`, `endpoints.html` | +| Web Search | Not available | Supported | `web-search.html`, `endpoints.html` | +| Projects / Workspaces | Default project only; use inference profiles for isolation and tagging | Create and manage your own | `projects.html`, `workspaces.html` | +| Cross-Region inference (Geo / Global profiles) | Supported | Not supported — in-Region only | `endpoints.html`, `monitoring-mantle-metrics.html` | +| In-Region processing | Through an on-demand (unprefixed) model ID where the model card lists one; proprietary GPT models list CRIS profiles only | Always in-Region | `endpoints.html`, the per-model cards | +| Guardrails, intelligent prompt routing | Supported | Not supported | `endpoints.html` | +| Application inference profiles / per-request metadata | InvokeModel and Converse only (Responses and Chat Completions reject an AIP target with 400) | Not available — Projects / Workspaces attribute usage | `cost-mgmt-application-inference-profiles.html`, `endpoints.html` | +| Model invocation logging | Supported, including the OpenAI-compatible APIs on this endpoint | Not currently captured | `model-invocation-logging.html` | +| CloudWatch metrics namespace | `AWS/Bedrock` (`Invocations`, `InputTokenCount`, …) | `AWS/BedrockMantle` (`Inferences`, `TotalInputTokens`, …) | `monitoring-mantle-metrics.html` | +| Authentication | SigV4 or Bedrock API key / short-term bearer token; IAM `bedrock:InvokeModel*` | SigV4 or Bedrock API key; IAM `bedrock-mantle:CreateInference` | `endpoints.html` | + +## Lifecycle wording (the only sentence to repeat) + +Existing `bedrock-mantle` applications continue to be fully supported and do not need to change. +Do not add claims about permanence, model-quality equivalence between the endpoints, +launch-day parity for new models, or future plans — the public pages make none, and the +guides in this plugin must not either. Both endpoint surfaces use the same underlying inference +engine (`endpoints.html`). + +## Quotas (why endpoint choice affects capacity planning) + +- **`bedrock-runtime`** — per-model, per-Region token quotas (input and output counted + together, output weighted by a model-specific burndown rate) shared across every inference + API on the endpoint; a requests-per-minute quota exists for some models and not others + (`quotas-runtime.html`, `scaling-throughput-best-practices.html`). +- **`bedrock-mantle`** — no requests-per-minute quota; separate input-tokens-per-minute and + output-tokens-per-minute quotas per model and Region for models with published quotas. A + model without a published quota is not unlimited — internal service capacity and transient + rate limiting still apply (`quotas-mantle.html`, `scaling-throughput-best-practices.html`). +- Traffic on the two endpoints consumes separate quota allocations even for the same model; + plan capacity per endpoint. Batch inference, Provisioned Throughput and custom inference + profiles exist only on `bedrock-runtime`. + +## Sources + +All pages under `https://docs.aws.amazon.com/bedrock/latest/userguide/`, read 2026-11-07: + +- `endpoints.html` — https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html +- `models-endpoint-availability.html` +- `inference-responses-api.html` +- `inference-chat-completions.html` +- `inference-messages-api.html` +- `tool-use-server-side.html` +- `web-search.html` +- `projects.html` +- `workspaces.html` +- `cost-mgmt-application-inference-profiles.html` +- `model-invocation-logging.html` +- `monitoring-mantle-metrics.html` +- `quotas-runtime.html` +- `quotas-mantle.html` +- `scaling-throughput-best-practices.html` diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md index c82a8eebc..820592554 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -175,26 +175,41 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` - **`bedrock_models[]`** — per Bedrock target: `source_model_id`, `aws_model_id`, `capabilities_matched[]`, `capability_gaps[]`, `honest_assessment`, `source_provider_price`, `bedrock_price`, `price_comparison`, `migration_complexity`, `model_change` (bool), - `migration_path` (see below), `quota_risk` (`low|medium|high`). Traditional-AI capabilities - do NOT appear here — they are `design_blocks[]` rows with `target_aws_service`. + `migration_path` (see below), `quota_risk` (`low|medium|high`), `required_client_api` + (`responses | chat_completions | converse | invoke | messages` — the client API surface the + generated code must use; for `runtime_openai_cris` it is the API the source app already + calls, never an implicit Converse), and `mantle_selector` (REQUIRED on every `mantle_*` row, + absent otherwise — see below). Traditional-AI capabilities do NOT appear here — they are + `design_blocks[]` rows with `target_aws_service`. - **`capability_mapping`** — per capability that is `true` in the profile's `integration.capabilities_summary`: `{ parity: "full|partial|none", notes }`. - **`code_migration`** — `primary_pattern` (matches profile `integration.pattern`), `framework`, `files_to_modify[]`, `dependency_changes`, and — when a retained source's `source_service` is `azure_openai` or a `provider`/`source_service` pair resolves to OpenAI — `openrouter_path` when a router was detected - (`same_model_mantle|direct|litellm|keep_openrouter`). + (`same_model_runtime|same_model_mantle|direct|litellm|keep_openrouter`; `same_model_runtime` + is the default same-model route — `bedrock-runtime` CRIS — and `same_model_mantle` requires a + `mantle_selector` on the row). - **`infrastructure[]`** — source-cloud AI infra resource → AWS equivalent, with confidence. - **`services_to_migrate[]`** — source-cloud AI service → AWS service, effort, notes. ## migration_path vocabulary -`mantle_openai_responses` (an OpenAI-protocol source — direct or Azure-fronted — with the model -on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `mantle_openai_chat` -(same, preserving a Chat Completions source on a model whose card lists Chat Completions on -mantle — GPT-6, GPT-5.5, GPT-5.4) · `runtime_openai_cris` (same-model move served via -`bedrock-runtime` with a CRIS id, `model_change: false`; `us.`/`in.`/`global.` for GPT-5.6 Terra -and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4, per their model cards) · `converse` (Bedrock-native +**`bedrock-runtime` is the default endpoint for every new path** (`vendored/ai/bedrock-endpoint-policy.md`): +a same-model OpenAI-protocol source records `runtime_openai_cris`, an Anthropic SDK source records +`runtime_messages`, and a `mantle_*` path is recorded only with a `bedrock_models[].mantle_selector`. + +`runtime_openai_cris` (same-model move served via `bedrock-runtime` `/openai/v1` with a CRIS id, +`model_change: false`, `required_client_api` = the source app's `responses` or `chat_completions`; +`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, +GPT-5.5 and GPT-5.4, per their model cards) · `runtime_messages` (Anthropic SDK kept on +`bedrock-runtime` `/anthropic` with a Claude profile id, `required_client_api: "messages"`, +`model_change` per the model mapping) · `mantle_openai_responses` (an OpenAI-protocol source — direct +or Azure-fronted — with the model on Bedrock and the region carrying it: keep the SDK, +`model_change: false`; requires `mantle_selector`) · `mantle_openai_chat` (same, preserving a Chat +Completions source on a model whose card lists Chat Completions on mantle — GPT-6, GPT-5.5, +GPT-5.4; requires `mantle_selector`) · `mantle_messages` (Anthropic SDK on the Mantle `/anthropic` +endpoint; requires `mantle_selector`, normally `workspaces`) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, intelligent prompt routing, or another feature no same-model path serves) · `gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK @@ -203,6 +218,14 @@ swap). A **bare** proprietary `openai.gpt-*` model ID is NEVER paired with a `bedrock-runtime` carries a CRIS-prefixed id (there is no runtime in-region form for GPT-6, GPT-5.6, GPT-5.5 or GPT-5.4). +## mantle_selector (required on every `mantle_*` `bedrock_models[]` row) + +One of `background_inference | server_side_tools | web_search | projects | workspaces | +mantle_only_model_or_api | in_region_processing` — the verified Mantle-only need that justified +leaving the `bedrock-runtime` default (definitions in `vendored/ai/bedrock-endpoint-policy.md`). +API continuity (the app already calls Responses, Chat Completions or Messages) is not a selector. +Absent on `runtime_*`, `converse`, `gpt-oss` and `direct` rows. + ## regional_validation `checked` (`aws___get_regional_availability` on the AWS MCP Server confirmed model/region @@ -267,7 +290,10 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4). - [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves to direct OpenAI: every source model that is available on Bedrock and carried by the - target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. + target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute — + and records `migration_path: "runtime_openai_cris"` by default. +- [ ] Every `bedrock_models[]` row whose `migration_path` starts with `mantle_` carries a + `mantle_selector` from the enum above; no `runtime_*`/`converse` row carries one. - [ ] `regional_warnings` and `multi_model_warnings` are present (`[]` when clean). - [ ] If `agentic_profile.is_agentic == true`: `agentic_design` is present with `migration_approach` matching `preferences.json`; a `harness` source whose origin resolves diff --git a/plugins/aws-startup-advisor/skills/shared/ai/README.md b/plugins/aws-startup-advisor/skills/shared/ai/README.md index d0d9088e6..2113c122e 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/README.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/README.md @@ -12,6 +12,7 @@ before opening a pull request. | ------------------------------------ | -------------------------------------------------------------- | | `ai-model-lifecycle.md` | Bedrock Active/Legacy/EOL registry + the 90-day exclusion rule | | `ai-migration-guardrails.md` | Shared constraints for every agentic migration path | +| `bedrock-endpoint-policy.md` | `bedrock-runtime` default / `bedrock-mantle` selector rule, `mantle_selector` enum, public endpoint matrix | | `bedrock-quotas.md` | TPM/RPM quota-risk assessment (works without an AWS account) | | `ai-openai-to-bedrock.md` | OpenAI-protocol (incl. Azure OpenAI) → Bedrock model selection | | `ai-anthropic-to-bedrock.md` | Anthropic SDK → Bedrock Converse client swap | diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md index dba311abf..852c8e2fb 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md @@ -83,11 +83,19 @@ GPT-5.4, the target is **the same model on Bedrock**. | GPT-5.4 | GPT-5.4 | `openai.gpt-5.4` / `us.` `global.` prefixed | `strong_migrate` — same model; ~10% over OpenAI std In-Region only (no Geo/Global price published) | Then apply the **region gate** below. Record `model_change: false` and the chosen path in -`aws-design-ai.json` → `ai_architecture.code_migration`: `migration_path: "mantle_openai_chat"` for -sources using Chat Completions on a model whose card lists it on mantle (GPT-6, GPT-5.5, GPT-5.4), or -`"mantle_openai_responses"` for a selected Mantle Responses path. Use `migration_path: "runtime_openai_cris"` -when a GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 target is served via `bedrock-runtime` with a CRIS id (the model -cards recommend runtime for new applications). +`aws-design-ai.json` → `ai_architecture.code_migration`. **The default for every Tier 0 target is +`migration_path: "runtime_openai_cris"`** — the same model served via `bedrock-runtime` with a CRIS +id (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, +GPT-5.5 and GPT-5.4) — with +`bedrock_models[].required_client_api` set to the API the source app already calls (`responses` or +`chat_completions`; `converse` only when a Converse-only feature such as Guardrails is required). +Amazon Bedrock recommends `bedrock-runtime` for new applications +(`references/vendored/ai/bedrock-endpoint-policy.md`). Record `migration_path: "mantle_openai_chat"` +(Chat Completions source on a model whose card lists it on mantle — GPT-6, GPT-5.5, GPT-5.4) or +`"mantle_openai_responses"` **only** when the row also records a `bedrock_models[].mantle_selector` +from that policy file (background inference, server-side tools / Web Search, Projects / Workspaces, +a Mantle-only model or API, or strict in-Region processing where runtime offers only CRIS). API +continuity alone never selects Mantle. Report the assessment honestly: `strong_migrate` here means "low-risk, well-supported move", not "cheaper". @@ -102,26 +110,30 @@ If neither path meets the constraints, offer a supported region or a model chang Do not assume GPT-5.6's broader regional footprint applies to Astra. **GPT-6 Astra / Sol / Luna / 6.1 Sol / 6.1 Terra:** same shape as GPT-5.6 — if the target region is in the mantle in-region matrix (narrower than GPT-5.6's: us-east-1 + us-west-2 for Astra only, us-east-1 only for Sol/Luna/6.1 Sol/6.1 Terra — see -`openai-on-bedrock.md` § Regional Availability), either endpoint works; otherwise the `bedrock-runtime` CRIS path -(`us.`/`global.` prefixed) covers broad commercial regions. GPT-6 Astra's CRIS path additionally lacks `Invoke` +`openai-on-bedrock.md` § Regional Availability), both endpoints work and the `bedrock-runtime` CRIS path +(`us.`/`global.` prefixed) is chosen unless a `mantle_selector` applies; outside that matrix the same CRIS path +covers broad commercial regions. GPT-6 Astra's CRIS path additionally lacks `Invoke` support (Responses/Chat Completions/Converse only) — verify per-feature before assuming parity with Sol/Luna/6.1 Sol. -**GPT-5.6 Sol / Terra / Luna:** rarely blocked. If the target region is in the mantle in-region matrix, either -endpoint works. If not, the `bedrock-runtime` CRIS path (Geo `us.`/`in.`, Global `global.`) covers most commercial -regions — offer it, stating the data-residency implication of cross-region routing (Geo stays within the geography; -Global routes anywhere and is the cost-parity option). Only a workload that requires strictly in-region processing -in a region outside the mantle matrix falls through to Tier 1. +**GPT-5.6 Sol / Terra / Luna:** rarely blocked. The `bedrock-runtime` CRIS path (Geo `us.` — plus `in.` for Terra +and Luna only — and Global `global.`) covers most commercial regions and is the default whether or not the target region is also in the +mantle in-region matrix — state the data-residency implication of cross-region routing (Geo stays within the +geography; Global routes anywhere and is the cost-parity option). A workload that requires strictly in-region +processing in a region inside the mantle matrix records `mantle_selector: "in_region_processing"` and takes the +Mantle path; one that requires it in a region outside the mantle matrix falls through to Tier 1. **GPT-5.5 / GPT-5.4:** same shape as GPT-5.6, with a narrower mantle matrix (GPT-5.5: us-east-1, us-east-2; -GPT-5.4: us-east-1, us-east-2, us-west-2, us-gov-west-1 — per their model cards, read 2026-11-06). If the target -region is in that matrix, either endpoint works. Otherwise the `bedrock-runtime` CRIS path (`us.openai.gpt-5.5` / -`global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`) reaches it, with the same data-residency -caveat as GPT-5.6 (Geo stays within the geography, Global routes anywhere). There is no runtime in-region form. -Only a workload that requires strictly in-region processing in a region outside the mantle matrix falls through: - -1. Prefer offering a **region change** to a supported mantle region, if no data-residency constraint pins the - workload. This preserves the zero-behavior-delta benefit, which is the whole point of Tier 0. +GPT-5.4: us-east-1, us-east-2, us-west-2, us-gov-west-1 — per their model cards, read 2026-11-06). The +`bedrock-runtime` CRIS path (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / +`global.openai.gpt-5.4`) is the default, with the same data-residency caveat as GPT-5.6 (Geo stays within the +geography, Global routes anywhere). There is no runtime in-region form. Strict in-region processing inside the +mantle matrix is a valid `in_region_processing` selector for the Mantle path. Only a workload that requires +strictly in-region processing in a region outside the mantle matrix falls through: + +1. Prefer offering a **region change** to a supported mantle region (recorded with + `mantle_selector: "in_region_processing"`), if no data-residency constraint pins the workload. This preserves + the zero-behavior-delta benefit, which is the whole point of Tier 0. 2. Otherwise fall through to Tier 1 and say why explicitly — a region constraint, not a model judgment. Record the outcome in `regional_warnings[]`. @@ -290,11 +302,11 @@ rather than dismissing it. For a long-context (>272K) workload the gap is much w | OpenAI Feature | Bedrock Equivalent | Notes | | ------------------------------------------ | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | -| OpenAI SDK (direct) | Same model on mantle Responses API | Base URL + credential + model ID change; see `shared/openai-on-bedrock.md` for the `/openai/v1` path | -| Responses API | Mantle Responses API | Closest to a drop-in; still verify the path and model ID | +| OpenAI SDK (direct) | Same model on `bedrock-runtime` `/openai/v1` (CRIS id) | Base URL + credential + model ID change; see `shared/openai-on-bedrock.md` for the `/openai/v1` path; Mantle only with a `mantle_selector` | +| Responses API | Runtime Responses API (`/openai/v1/responses`) | Closest to a drop-in; always synchronous on runtime (`background=true` needs a Mantle selector) | | Chat Completions | Preserve supported Chat when selected | Reshape only for a selected API change; verify the model and endpoint | -| Function calling | Supported on GPT-5.x via mantle; Claude tools via Converse | Same-model keeps tool semantics identical | -| Reasoning effort | `reasoning={"effort": ...}` on mantle | `none` / `low` / `medium` / `high` / `xhigh` / `max` | +| Function calling | Supported on GPT-5.x via runtime or mantle; Claude tools via Converse | Same-model keeps tool semantics identical; server-side tools are Mantle-only | +| Reasoning effort | `reasoning={"effort": ...}` on either endpoint | `none` / `low` / `medium` / `high` / `xhigh` / `max` | | Prompt caching | GPT-5.6 (90% off cached input); GPT-5.5 / 5.4 implicit caching on `bedrock-runtime` Responses only | GPT-5.5 / 5.4 cards (read 2026-11-06) list a cache-read rate and implicit caching on runtime, none on mantle; Claude has its own caching | | Streaming | Supported | Verify per surface | | Vision | GPT-5.x (image input) or Claude / Llama 4 | Same-model preserves behavior | @@ -306,22 +318,34 @@ rather than dismissing it. For a long-context (>272K) workload the gap is much w | Assistants API | See decision tree below | Path depends on which features are used | | Realtime API | No equivalent | Stay on OpenAI for this | | Codex | Not verified on Bedrock — see `shared/openai-on-bedrock.md` | Do not price; re-check the OpenAI model card index first | -| Guardrails / KB / invocation logging | Bedrock-native model or gpt-oss | Not available through the mantle GPT path | +| Guardrails / KB / invocation logging | `bedrock-runtime` CRIS path (Guardrails via Converse; logging on every runtime API); KB needs a Bedrock-native model or gpt-oss | None of these are available through the mantle GPT path | --- ## Common Migration Paths -### Same model on Bedrock (Tier 0) — smallest possible change +### Same model on Bedrock (Tier 0) — `bedrock-runtime` by default -The application keeps the OpenAI SDK and the same model. Changes are limited to: +The application keeps the OpenAI SDK, the same model and the same API it calls today +(`migration_path: "runtime_openai_cris"`, `required_client_api` = `responses` or `chat_completions`). Changes are +limited to: -1. Base URL → `https://bedrock-mantle.{region}.api.aws/openai/v1` (note the `openai/v1` segment) -2. Credentials → Bedrock API key or the auto-refreshing `BedrockOpenAI` token provider, **not** an OpenAI key -3. Model ID → the `openai.gpt-*` form -4. IAM → `bedrock-mantle:*` actions, e.g. via `AmazonBedrockMantleInferenceAccess` +1. Base URL → `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` (note the `openai/v1` segment) +2. Credentials → a short-term Bedrock bearer token (`api_key=lambda: provide_token(region=region)` from + `aws-bedrock-token-generator`) or a Bedrock API key, **not** an OpenAI key +3. Model ID → the CRIS inference-profile id (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only + for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), never the bare `openai.gpt-*` form +4. IAM → `bedrock:InvokeModel*` on the inference-profile and foundation-model ARNs (GPT-6 / GPT-5.5 / GPT-5.4 + also need `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account-id}:project/default`), plus + `bedrock:GetInferenceProfile` on the profile -Full snippet and prerequisites in `shared/openai-on-bedrock.md`. No prompt changes, no eval regression expected — +**Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` +(`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, +the auto-refreshing `BedrockOpenAI` token provider or a Bedrock API key, the bare `openai.gpt-*` id, and +`bedrock-mantle:*` IAM (e.g. `AmazonBedrockMantleInferenceAccess`). Record `migration_path: "mantle_openai_responses"` +or `"mantle_openai_chat"` with the selector. + +Full snippets and prerequisites in `shared/openai-on-bedrock.md`. No prompt changes, no eval regression expected — but still run the eval harness to confirm, because infrastructure and tokenizer-adjacent behavior can differ. ### Assistants API → migration decision tree @@ -329,16 +353,17 @@ but still run the eval harness to confirm, because infrastructure and tokenizer- Assistants API and Responses API are different surfaces. Do not treat all Assistants usage as a config-only change. **1. App already uses the OpenAI Responses API** (`responses.create`) -→ Cleanest path. Base URL + credential + model ID change. +→ Cleanest path. Base URL + credential + model ID change on `bedrock-runtime` (`runtime_openai_cris`). **2. App uses Assistants API only for stateful multi-turn conversation** (no hosted tools, no file search, no code interpreter, no persistent Assistant objects, no complex run lifecycle) -→ Mantle Responses is viable. Requires migrating `threads`/`runs` to `responses.create` — a small API migration -(days), not a redesign. Not a zero-code change. +→ Runtime Responses is viable (stateful `previous_response_id` works on `bedrock-runtime`). Requires migrating +`threads`/`runs` to `responses.create` — a small API migration (days), not a redesign. Not a zero-code change. **3. App uses Assistants API with simple hosted tools** (function calling only) -→ Mantle Responses with tool use is viable. Moderate migration (1–2 weeks) to adapt tool definitions and the run -lifecycle. +→ Runtime Responses with client-side tool use is viable. Moderate migration (1–2 weeks) to adapt tool definitions +and the run lifecycle. Server-side or pre-configured tools (including Web Search) are a `server_side_tools` / +`web_search` Mantle selector. **4. App uses Assistants API with file search, vector stores, code interpreter, persistent Assistant objects, or complex run lifecycle management** @@ -347,15 +372,16 @@ Gateway, gateway-fronted knowledge bases) for a full agentic replacement (2–4 if the team prefers to own state. Never target classic Bedrock Agents (`bedrock-agent`) — it is in maintenance mode and closed to new customers as of July 30, 2026. -### When to prefer a Bedrock-native model over the mantle GPT path +### When to prefer a Bedrock-native model over the same-model GPT path -Choose Claude / Nova via Converse when the workload needs Knowledge Bases, intelligent prompt routing, structured -outputs, or application inference profiles — none of which the mantle GPT path offers. Note what no longer forces -a family switch for GPT-5.6, GPT-5.5 and GPT-5.4: their `bedrock-runtime` CRIS paths provide cross-region -inference, invocation logs, Guardrails and application inference profiles (both Converse API only), and -structured outputs and implicit prompt caching (both Responses API only) — per the GPT-5.4 / GPT-5.5 model cards -read 2026-11-06. Knowledge Bases and intelligent prompt routing still require a Bedrock-native model. The mantle -GPT path is the smallest change; the Bedrock-native path remains the most feature-complete. +Choose Claude / Nova via Converse when the workload needs Knowledge Bases or intelligent prompt routing — neither +is served by a same-model GPT path on either endpoint. Note what no longer forces a family switch for GPT-5.6, +GPT-5.5 and GPT-5.4: their `bedrock-runtime` CRIS paths (the default) provide cross-region inference, invocation +logs, Guardrails and application inference profiles (both Converse API only), and structured outputs and implicit +prompt caching (both Responses API only) — per the GPT-5.4 / GPT-5.5 model cards read 2026-11-06. Knowledge +Bases and intelligent prompt routing still require a Bedrock-native model. The same-model `bedrock-runtime` path +is the default; the Bedrock-native path is the most feature-complete; the mantle GPT path is reserved for a +recorded `mantle_selector`. ### High-spend tiering diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-openrouter-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-openrouter-to-bedrock.md index ca91e4173..18ff0e281 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-openrouter-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-openrouter-to-bedrock.md @@ -62,7 +62,7 @@ For every distinct upstream model behind the gateway: 1. **Select the provider guide** from the provider segment (`anthropic` / `openai` / `google`). 2. **Apply that guide's selection policy** to the exact model segment — same-model-on-Bedrock where it exists (OpenAI's Tier 0, Claude version-to-version), else the guide's cross-family / tier mapping. The provider guides own all pricing tables, region gates, and lifecycle checks; do not restate them here. -3. **Record the mapping per upstream, including the guide's selected endpoint** — the gateway's output is a set of `{ upstream_model → bedrock_target }` decisions, not one decision, and each decision carries the `migration_path`, endpoint/inference-profile ID, and credential/IAM requirement the provider guide chose. For GPT targets `ai-openai-to-bedrock.md` may select the Mantle Responses path (`mantle_openai_responses`) **or** the `bedrock-runtime` CRIS path (`runtime_openai_cris`, with a `us.`/`global.` profile) depending on the model and region gate — carry whichever it selected through to cutover, do not collapse every GPT target onto Mantle. Two Claude models behind the same gateway produce two rows, never one. +3. **Record the mapping per upstream, including the guide's selected endpoint** — the gateway's output is a set of `{ upstream_model → bedrock_target }` decisions, not one decision, and each decision carries the `migration_path`, endpoint/inference-profile ID, and credential/IAM requirement the provider guide chose. For GPT targets `ai-openai-to-bedrock.md` selects the `bedrock-runtime` CRIS path by default (`runtime_openai_cris`, with a `us.`/`global.` profile and the source app's `required_client_api`) and the Mantle path (`mantle_openai_responses` / `mantle_openai_chat`) only with a recorded `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`) — carry whichever it selected through to cutover, do not collapse every GPT target onto Mantle. Two Claude models behind the same gateway produce two rows, never one. Preserve the distinction between a provider reached **through OpenRouter** and the **same provider reached directly**. A workload may use direct Anthropic SDK calls *and* Anthropic-via-OpenRouter at once; both map to Bedrock-hosted Claude, but they are different source integrations with different cutover work (a direct SDK swap vs. removing a gateway hop). @@ -72,8 +72,8 @@ Preserve the distinction between a provider reached **through OpenRouter** and t Replacing OpenRouter with Bedrock is an integration change, not just a model change: -- **Transport — per the selected `migration_path`, not one default.** For Claude / Nova / Llama / Gemini-replacement targets, the OpenRouter client becomes the Bedrock Converse API. For GPT targets the transport follows the provider guide's selection: the Bedrock Mantle OpenAI-compatible endpoint when it chose `mantle_openai_responses`, or `bedrock-runtime` with the selected CRIS inference profile when it chose `runtime_openai_cris`. Do not force every GPT target onto Mantle — a runtime/CRIS selection sent to Mantle (or a Mantle-only model routed to runtime) is an incompatible cutover plan. A workload that used OpenRouter precisely for its one-API-many-models convenience may want a thin internal router over Bedrock to preserve that shape — see `references/vendored/ai/ai-migration-guardrails.md`. -- **Credentials.** Remove the OpenRouter key and any BYOK upstream keys; use IAM matched to each target's selected path — `bedrock:InvokeModel` for Converse and the `runtime_openai_cris` path, `bedrock-mantle:*` (e.g. `AmazonBedrockMantleInferenceAccess`) for the Mantle GPT path. The source-key retirement rules apply — do not plant the old gateway/provider keys in the target. +- **Transport — per the selected `migration_path`; the default is `bedrock-runtime`.** For Claude / Nova / Llama / Gemini-replacement targets, the OpenRouter client becomes the Bedrock Converse API. For GPT targets the transport follows the provider guide's selection: `bedrock-runtime` `/openai/v1` with the selected CRIS inference profile when it chose `runtime_openai_cris` (the default — recorded as `openrouter_path: "same_model_runtime"`), or the Bedrock Mantle OpenAI-compatible endpoint only when it chose `mantle_openai_responses` / `mantle_openai_chat` with a `mantle_selector` (`openrouter_path: "same_model_mantle"`). Do not force every GPT target onto Mantle — a runtime/CRIS selection sent to Mantle (or a Mantle-only model routed to runtime) is an incompatible cutover plan. A workload that used OpenRouter precisely for its one-API-many-models convenience may want a thin internal router over Bedrock to preserve that shape — see `references/vendored/ai/ai-migration-guardrails.md`. +- **Credentials.** Remove the OpenRouter key and any BYOK upstream keys; use IAM matched to each target's selected path — `bedrock:InvokeModel*` for Converse and the `runtime_openai_cris` path (the default), `bedrock-mantle:*` (e.g. `AmazonBedrockMantleInferenceAccess`) only for a selector-gated Mantle GPT path. The source-key retirement rules apply — do not plant the old gateway/provider keys in the target. - **Routing policy.** If the app used OpenRouter's automatic model selection or fallback routing, that policy has no Bedrock equivalent and must be reimplemented in the application or dropped deliberately — flag it rather than assuming parity. - **Per-upstream validation.** Validate each mapped target independently (behavior, tool-calling, structured output) the way its provider guide prescribes; a gateway migration is only complete when every retained upstream passes. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md new file mode 100644 index 000000000..fdc4948a2 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md @@ -0,0 +1,115 @@ +# Bedrock Endpoint Policy — `bedrock-runtime` by default, `bedrock-mantle` by selector + +> Canonical endpoint-selection policy for every Bedrock migration path in this plugin. +> Vendored into each consuming skill as `references/vendored/ai/bedrock-endpoint-policy.md` +> and kept byte-identical by `tools/sync-vendored.py`; edit HERE, then run the sync. +> Every statement below comes from the public Amazon Bedrock User Guide pages listed in +> § Sources (read 2026-11-07). Re-read those pages before changing a row. + +## The rule + +**Use `bedrock-runtime` for new applications.** Amazon Bedrock documents `bedrock-runtime` as the +recommended endpoint for most new applications; it serves InvokeModel, Converse, the +OpenAI-compatible Responses and Chat Completions APIs, and the Anthropic Messages API. + +**Use `bedrock-mantle` only for a verified Mantle-only need.** The documented reasons are: + +- background (asynchronous, long-running) inference — Responses `background=true` is rejected + with a 400 on `bedrock-runtime`; +- server-side tool use and pre-configured tools, including Web Search; +- customer-created Projects (OpenAI-compatible) or Workspaces (Anthropic-compatible) — + `bedrock-runtime` exposes only the account's default project; +- a model or API that the Endpoint availability page lists on `bedrock-mantle` only; +- strict in-Region processing for a model that `bedrock-runtime` serves only through a + cross-Region inference (CRIS) profile while `bedrock-mantle` serves it in the required Region. + +**API continuity alone never selects Mantle.** An OpenAI Responses, OpenAI Chat Completions or +Anthropic Messages source keeps its SDK on `bedrock-runtime` by changing the base URL, the +credential and the model ID. Choosing Mantle because "the app already speaks Responses" is not a +selector. + +**Pricing never selects an endpoint.** Per-token pricing for the same model is identical on both +endpoints; choose by API and capability need. + +## `mantle_selector` — the recorded reason + +Every `bedrock_models[]` row whose `migration_path` starts with `mantle_` (`mantle_openai_responses`, +`mantle_openai_chat`, `mantle_messages`) MUST record `mantle_selector` with exactly one value: + +| `mantle_selector` | Verified need | +| --------------------------- | ----------------------------------------------------------------------------------------------- | +| `background_inference` | Responses requests with `background=true` / long-running asynchronous inference | +| `server_side_tools` | Server-side tool use (Lambda-backed MCP tools) or pre-configured tools other than Web Search | +| `web_search` | The built-in Web Search tool | +| `projects` | Customer-created Projects (OpenAI-compatible) for isolation and cost tracking | +| `workspaces` | Workspaces (Anthropic-compatible) referenced by `anthropic-workspace-id` | +| `mantle_only_model_or_api` | The model, or the API for that model, is listed on `bedrock-mantle` only | +| `in_region_processing` | Strict in-Region processing where `bedrock-runtime` offers only a CRIS profile for that model | + +A `mantle_*` row without a `mantle_selector`, or with a reason outside this table, is a design +defect: rewrite the row as the `bedrock-runtime` path (`runtime_openai_cris` with the source +app's `required_client_api`, or `runtime_messages`) or justify it with one of the values above. +A `runtime_*` or `converse` row never carries `mantle_selector`. + +## Public comparison (read 2026-11-07) + +| Item | `bedrock-runtime` | `bedrock-mantle` | Source page | +| ---------------------------------------------- | --------------------------------------------------------------------------------- | --------------------------------------------------------------- | ----------------------------------------------- | +| Recommended default | Yes — "for most new applications" | Only for capabilities available only there | `endpoints.html`, `models-endpoint-availability.html` | +| InvokeModel / Converse | Supported | Not supported | `endpoints.html` | +| OpenAI Responses API | Supported at `/openai/v1`; always synchronous; default project only | Supported at `/v1` (some models `/openai/v1`); background OK | `inference-responses-api.html` | +| OpenAI Chat Completions API | Supported at `/openai/v1/chat/completions` | Supported at `/v1/chat/completions` (some models `/openai/v1`) | `inference-chat-completions.html` | +| Anthropic Messages API | Supported at `/anthropic` (SDK sends `/anthropic/v1/messages`) | Supported at `/anthropic/v1/messages` | `inference-messages-api.html` | +| Background Responses (`background=true`) | Rejected (400) | Supported | `endpoints.html`, `inference-responses-api.html` | +| Server-side tool use / pre-configured tools | Not available; client-side tools work | Supported | `tool-use-server-side.html`, `endpoints.html` | +| Web Search | Not available | Supported | `web-search.html`, `endpoints.html` | +| Projects / Workspaces | Default project only; use inference profiles for isolation and tagging | Create and manage your own | `projects.html`, `workspaces.html` | +| Cross-Region inference (Geo / Global profiles) | Supported | Not supported — in-Region only | `endpoints.html`, `monitoring-mantle-metrics.html` | +| In-Region processing | Through an on-demand (unprefixed) model ID where the model card lists one; proprietary GPT models list CRIS profiles only | Always in-Region | `endpoints.html`, the per-model cards | +| Guardrails, intelligent prompt routing | Supported | Not supported | `endpoints.html` | +| Application inference profiles / per-request metadata | InvokeModel and Converse only (Responses and Chat Completions reject an AIP target with 400) | Not available — Projects / Workspaces attribute usage | `cost-mgmt-application-inference-profiles.html`, `endpoints.html` | +| Model invocation logging | Supported, including the OpenAI-compatible APIs on this endpoint | Not currently captured | `model-invocation-logging.html` | +| CloudWatch metrics namespace | `AWS/Bedrock` (`Invocations`, `InputTokenCount`, …) | `AWS/BedrockMantle` (`Inferences`, `TotalInputTokens`, …) | `monitoring-mantle-metrics.html` | +| Authentication | SigV4 or Bedrock API key / short-term bearer token; IAM `bedrock:InvokeModel*` | SigV4 or Bedrock API key; IAM `bedrock-mantle:CreateInference` | `endpoints.html` | + +## Lifecycle wording (the only sentence to repeat) + +Existing `bedrock-mantle` applications continue to be fully supported and do not need to change. +Do not add claims about permanence, model-quality equivalence between the endpoints, +launch-day parity for new models, or future plans — the public pages make none, and the +guides in this plugin must not either. Both endpoint surfaces use the same underlying inference +engine (`endpoints.html`). + +## Quotas (why endpoint choice affects capacity planning) + +- **`bedrock-runtime`** — per-model, per-Region token quotas (input and output counted + together, output weighted by a model-specific burndown rate) shared across every inference + API on the endpoint; a requests-per-minute quota exists for some models and not others + (`quotas-runtime.html`, `scaling-throughput-best-practices.html`). +- **`bedrock-mantle`** — no requests-per-minute quota; separate input-tokens-per-minute and + output-tokens-per-minute quotas per model and Region for models with published quotas. A + model without a published quota is not unlimited — internal service capacity and transient + rate limiting still apply (`quotas-mantle.html`, `scaling-throughput-best-practices.html`). +- Traffic on the two endpoints consumes separate quota allocations even for the same model; + plan capacity per endpoint. Batch inference, Provisioned Throughput and custom inference + profiles exist only on `bedrock-runtime`. + +## Sources + +All pages under `https://docs.aws.amazon.com/bedrock/latest/userguide/`, read 2026-11-07: + +- `endpoints.html` — https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html +- `models-endpoint-availability.html` +- `inference-responses-api.html` +- `inference-chat-completions.html` +- `inference-messages-api.html` +- `tool-use-server-side.html` +- `web-search.html` +- `projects.html` +- `workspaces.html` +- `cost-mgmt-application-inference-profiles.html` +- `model-invocation-logging.html` +- `monitoring-mantle-metrics.html` +- `quotas-runtime.html` +- `quotas-mantle.html` +- `scaling-throughput-best-practices.html` diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md index c82a8eebc..820592554 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md @@ -175,26 +175,41 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` - **`bedrock_models[]`** — per Bedrock target: `source_model_id`, `aws_model_id`, `capabilities_matched[]`, `capability_gaps[]`, `honest_assessment`, `source_provider_price`, `bedrock_price`, `price_comparison`, `migration_complexity`, `model_change` (bool), - `migration_path` (see below), `quota_risk` (`low|medium|high`). Traditional-AI capabilities - do NOT appear here — they are `design_blocks[]` rows with `target_aws_service`. + `migration_path` (see below), `quota_risk` (`low|medium|high`), `required_client_api` + (`responses | chat_completions | converse | invoke | messages` — the client API surface the + generated code must use; for `runtime_openai_cris` it is the API the source app already + calls, never an implicit Converse), and `mantle_selector` (REQUIRED on every `mantle_*` row, + absent otherwise — see below). Traditional-AI capabilities do NOT appear here — they are + `design_blocks[]` rows with `target_aws_service`. - **`capability_mapping`** — per capability that is `true` in the profile's `integration.capabilities_summary`: `{ parity: "full|partial|none", notes }`. - **`code_migration`** — `primary_pattern` (matches profile `integration.pattern`), `framework`, `files_to_modify[]`, `dependency_changes`, and — when a retained source's `source_service` is `azure_openai` or a `provider`/`source_service` pair resolves to OpenAI — `openrouter_path` when a router was detected - (`same_model_mantle|direct|litellm|keep_openrouter`). + (`same_model_runtime|same_model_mantle|direct|litellm|keep_openrouter`; `same_model_runtime` + is the default same-model route — `bedrock-runtime` CRIS — and `same_model_mantle` requires a + `mantle_selector` on the row). - **`infrastructure[]`** — source-cloud AI infra resource → AWS equivalent, with confidence. - **`services_to_migrate[]`** — source-cloud AI service → AWS service, effort, notes. ## migration_path vocabulary -`mantle_openai_responses` (an OpenAI-protocol source — direct or Azure-fronted — with the model -on Bedrock and the region carrying it: keep the SDK, `model_change: false`) · `mantle_openai_chat` -(same, preserving a Chat Completions source on a model whose card lists Chat Completions on -mantle — GPT-6, GPT-5.5, GPT-5.4) · `runtime_openai_cris` (same-model move served via -`bedrock-runtime` with a CRIS id, `model_change: false`; `us.`/`in.`/`global.` for GPT-5.6 Terra -and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4, per their model cards) · `converse` (Bedrock-native +**`bedrock-runtime` is the default endpoint for every new path** (`vendored/ai/bedrock-endpoint-policy.md`): +a same-model OpenAI-protocol source records `runtime_openai_cris`, an Anthropic SDK source records +`runtime_messages`, and a `mantle_*` path is recorded only with a `bedrock_models[].mantle_selector`. + +`runtime_openai_cris` (same-model move served via `bedrock-runtime` `/openai/v1` with a CRIS id, +`model_change: false`, `required_client_api` = the source app's `responses` or `chat_completions`; +`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, +GPT-5.5 and GPT-5.4, per their model cards) · `runtime_messages` (Anthropic SDK kept on +`bedrock-runtime` `/anthropic` with a Claude profile id, `required_client_api: "messages"`, +`model_change` per the model mapping) · `mantle_openai_responses` (an OpenAI-protocol source — direct +or Azure-fronted — with the model on Bedrock and the region carrying it: keep the SDK, +`model_change: false`; requires `mantle_selector`) · `mantle_openai_chat` (same, preserving a Chat +Completions source on a model whose card lists Chat Completions on mantle — GPT-6, GPT-5.5, +GPT-5.4; requires `mantle_selector`) · `mantle_messages` (Anthropic SDK on the Mantle `/anthropic` +endpoint; requires `mantle_selector`, normally `workspaces`) · `converse` (Bedrock-native Converse/`bedrock-runtime`; set `model_change: true` when the workload needs Knowledge Bases, intelligent prompt routing, or another feature no same-model path serves) · `gpt-oss` (OpenAI-lineage model on the Bedrock-native runtime) · `direct` (framework-agnostic Bedrock SDK @@ -203,6 +218,14 @@ swap). A **bare** proprietary `openai.gpt-*` model ID is NEVER paired with a `bedrock-runtime` carries a CRIS-prefixed id (there is no runtime in-region form for GPT-6, GPT-5.6, GPT-5.5 or GPT-5.4). +## mantle_selector (required on every `mantle_*` `bedrock_models[]` row) + +One of `background_inference | server_side_tools | web_search | projects | workspaces | +mantle_only_model_or_api | in_region_processing` — the verified Mantle-only need that justified +leaving the `bedrock-runtime` default (definitions in `vendored/ai/bedrock-endpoint-policy.md`). +API continuity (the app already calls Responses, Chat Completions or Messages) is not a selector. +Absent on `runtime_*`, `converse`, `gpt-oss` and `direct` rows. + ## regional_validation `checked` (`aws___get_regional_availability` on the AWS MCP Server confirmed model/region @@ -267,7 +290,10 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4). - [ ] For a retained source whose `source_service` is `azure_openai`, or whose provider resolves to direct OpenAI: every source model that is available on Bedrock and carried by the - target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute. + target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute — + and records `migration_path: "runtime_openai_cris"` by default. +- [ ] Every `bedrock_models[]` row whose `migration_path` starts with `mantle_` carries a + `mantle_selector` from the enum above; no `runtime_*`/`converse` row carries one. - [ ] `regional_warnings` and `multi_model_warnings` are present (`[]` when clean). - [ ] If `agentic_profile.is_agentic == true`: `agentic_design` is present with `migration_approach` matching `preferences.json`; a `harness` source whose origin resolves diff --git a/plugins/aws-startup-advisor/tests/test_ai_only_option_c_framework_route.py b/plugins/aws-startup-advisor/tests/test_ai_only_option_c_framework_route.py index 67e767ca1..c860fca07 100644 --- a/plugins/aws-startup-advisor/tests/test_ai_only_option_c_framework_route.py +++ b/plugins/aws-startup-advisor/tests/test_ai_only_option_c_framework_route.py @@ -119,3 +119,26 @@ def test_dd38bc2_spellings_and_openrouter_priority_preserved() -> None: f"{f.name}: the OpenRouter subroute no longer takes priority over " f"migration_path (dd38bc2 ordering regressed)" ) + + +RUNTIME_LABEL = "[C] Generate the Bedrock runtime (CRIS) migration script and A/B test harness" + + +def test_runtime_subroutes_resolve_to_runtime_label() -> None: + # bedrock-runtime is the default same-model endpoint, so both menus must + # resolve the runtime OpenRouter subroute and the runtime_openai_cris + # migration_path to the runtime label, not the Mantle one. + for f in AI_ONLY_FILES: + section = _ai_only_section(f.read_text(encoding="utf-8")) + assert "same_model_runtime" in section, ( + f"{f.name}: the OpenRouter subroute does not know same_model_runtime" + ) + assert "runtime_openai_cris" in section, ( + f"{f.name}: migration_path resolution does not know runtime_openai_cris" + ) + assert RUNTIME_LABEL in section, ( + f"{f.name}: the runtime (CRIS) option-C label is missing" + ) + # Mantle label remains for the selector-gated routes. + assert "[C] Generate the Mantle migration script and A/B test harness" in section + diff --git a/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py b/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py new file mode 100644 index 000000000..1760e0cb6 --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py @@ -0,0 +1,113 @@ +"""Guard: the canonical Bedrock endpoint policy and its consumers. + +Amazon Bedrock documents `bedrock-runtime` as the recommended endpoint for most +new applications and `bedrock-mantle` for capabilities available only there +(https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html, read +2026-11-07). The plugin states that rule once, in +`skills/shared/ai/bedrock-endpoint-policy.md`, vendors it into gcp-to-aws and +azure-to-aws, and requires every `mantle_*` design row to record a +`mantle_selector`. These checks pin the policy file's contract and the design +files that must point at it. The llm-to-bedrock SKILL.md reference is added by +the execution-lane change and is asserted there, not here. +""" +from __future__ import annotations + +import re +from pathlib import Path + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent + +POLICY_CANONICAL = PLUGIN_ROOT / "skills" / "shared" / "ai" / "bedrock-endpoint-policy.md" +POLICY_VENDORED = [ + PLUGIN_ROOT / "skills" / "gcp-to-aws" / "references" / "vendored" / "ai" / "bedrock-endpoint-policy.md", + PLUGIN_ROOT / "skills" / "azure-to-aws" / "references" / "vendored" / "ai" / "bedrock-endpoint-policy.md", +] + +DESIGN_AI_FILES = [ + PLUGIN_ROOT / "skills" / "gcp-to-aws" / "references" / "phases" / "design" / "design-ai.md", + PLUGIN_ROOT / "skills" / "azure-to-aws" / "references" / "phases" / "design" / "design-ai.md", +] + +SCHEMA_FILES = [ + PLUGIN_ROOT / "skills" / "shared" / "ai" / "schema-design-aws-ai.md", + PLUGIN_ROOT / "skills" / "gcp-to-aws" / "references" / "shared" / "schema-design-aws-ai.md", + PLUGIN_ROOT / "skills" / "azure-to-aws" / "references" / "shared" / "schema-design-aws-ai.md", +] + +MANTLE_SELECTOR_ENUM = [ + "background_inference", + "server_side_tools", + "web_search", + "projects", + "workspaces", + "mantle_only_model_or_api", + "in_region_processing", +] + +ENDPOINTS_URL = "https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html" + +# Claims the public pages do not make and the policy must not add. +FORBIDDEN_WORDING = ["not deprecated", "guarantee", "roadmap"] + + +def _norm(text: str) -> str: + return re.sub(r"\s+", " ", text) + + +def test_policy_file_exists_and_vendored_copies_are_byte_identical() -> None: + assert POLICY_CANONICAL.is_file(), "skills/shared/ai/bedrock-endpoint-policy.md is missing" + canonical = POLICY_CANONICAL.read_bytes() + for vendored in POLICY_VENDORED: + assert vendored.is_file(), f"{vendored} missing — add the README row and run tools/sync-vendored.py" + assert vendored.read_bytes() == canonical, f"{vendored} drifted from the canonical policy" + + +def test_policy_states_the_runtime_default_and_selector_rule() -> None: + text = _norm(POLICY_CANONICAL.read_text(encoding="utf-8")) + assert "Use `bedrock-runtime` for new applications" in text + assert "API continuity alone never selects Mantle" in text + for value in MANTLE_SELECTOR_ENUM: + assert f"`{value}`" in text, f"mantle_selector enum value {value} missing from the policy" + assert ENDPOINTS_URL in text, "policy must cite the public endpoints.html page" + assert re.search(r"read 20\d\d-\d\d-\d\d", text), "policy must carry a 'read ' token" + + +def test_policy_avoids_lifecycle_and_roadmap_claims() -> None: + text = POLICY_CANONICAL.read_text(encoding="utf-8").lower() + for phrase in FORBIDDEN_WORDING: + assert phrase not in text, f"policy must not claim '{phrase}'" + assert "continue to be fully supported and do not need to change" in text + + +def test_design_ai_files_reference_the_policy() -> None: + for f in DESIGN_AI_FILES: + text = f.read_text(encoding="utf-8") + assert "bedrock-endpoint-policy.md" in text, f"{f}: does not point at the endpoint policy" + assert "mantle_selector" in text, f"{f}: does not require mantle_selector on mantle_* rows" + + +def test_schema_docs_carry_the_runtime_default_vocabulary() -> None: + for f in SCHEMA_FILES: + text = f.read_text(encoding="utf-8") + assert "bedrock-endpoint-policy.md" in text, f"{f}: does not point at the endpoint policy" + assert "runtime_messages" in text, f"{f}: migration_path vocabulary missing runtime_messages" + assert "same_model_runtime" in text, f"{f}: openrouter_path vocabulary missing same_model_runtime" + for value in MANTLE_SELECTOR_ENUM: + assert value in text, f"{f}: mantle_selector enum value {value} missing" + + +def test_fixtures_record_a_selector_on_every_mantle_row() -> None: + import json + + for fixture in (PLUGIN_ROOT / "fixtures" / "aws-design-ai").glob("*/aws-design-ai.json"): + design = json.loads(fixture.read_text(encoding="utf-8")) + for row in design.get("ai_architecture", {}).get("bedrock_models", []): + path = row.get("migration_path", "") + if path.startswith("mantle"): + assert row.get("mantle_selector") in MANTLE_SELECTOR_ENUM, ( + f"{fixture}: mantle row {row.get('aws_model_id')} lacks a valid mantle_selector" + ) + else: + assert "mantle_selector" not in row, ( + f"{fixture}: non-mantle row {row.get('aws_model_id')} must not carry mantle_selector" + ) diff --git a/plugins/aws-startup-advisor/tests/test_generate_mantle_dispatch.py b/plugins/aws-startup-advisor/tests/test_generate_mantle_dispatch.py index 92722d8fd..f08bbd5a6 100644 --- a/plugins/aws-startup-advisor/tests/test_generate_mantle_dispatch.py +++ b/plugins/aws-startup-advisor/tests/test_generate_mantle_dispatch.py @@ -122,3 +122,29 @@ def test_azure_generate_mantle_branch_still_correct() -> None: "Azure generate-artifacts-ai.md regressed off the bedrock-mantle:* IAM " "action for the Mantle path" ) + # Runtime-default: Mantle is no longer presented as Azure's primary path. + assert "Mantle is the PRIMARY path" not in text, ( + "Azure generate-artifacts-ai.md still calls Mantle the PRIMARY path; " + "bedrock-runtime CRIS is the default and Mantle is selector-gated" + ) + assert "bedrock-runtime" in text + + +def test_step0_runtime_cris_is_default_and_mantle_needs_selector() -> None: + # bedrock-runtime is the documented default endpoint (endpoints.html); Step 0 + # must list runtime_openai_cris before the mantle bullet, label it the default, + # and tie the mantle bullet to a recorded mantle_selector. Step 3B's OpenRouter + # subroute must know the runtime same-model route. + text = GCP_GENERATE.read_text(encoding="utf-8") + step0 = text.split("## Step 0:", 1)[1].split("## Step 1M:", 1)[0] + runtime_idx = step0.index("`migration_path` = `\"runtime_openai_cris\"`") + mantle_idx = step0.index("`migration_path` starts with `\"mantle\"`") + assert runtime_idx < mantle_idx, ( + "Step 0 must list the runtime_openai_cris bullet before the mantle bullet" + ) + assert "default" in step0[runtime_idx:mantle_idx].lower() + assert "mantle_selector" in step0 + assert "bedrock-endpoint-policy.md" in step0 + step3b = text.split("## Step 3B: Generate Gateway Configuration", 1)[1] + assert "same_model_runtime" in step3b + assert "same_model_mantle" in step3b diff --git a/plugins/aws-startup-advisor/tests/test_gpt6_review_round2.py b/plugins/aws-startup-advisor/tests/test_gpt6_review_round2.py index 2309b54cb..60597fabc 100644 --- a/plugins/aws-startup-advisor/tests/test_gpt6_review_round2.py +++ b/plugins/aws-startup-advisor/tests/test_gpt6_review_round2.py @@ -89,6 +89,14 @@ def test_design_ai_gives_gpt55_gpt54_the_runtime_cris_path() -> None: assert "GPT-5.5 and GPT-5.4 are `bedrock-mantle` only" not in text assert "no CRIS" not in text assert "No Converse fallback for GPT-5.5/GPT-5.4" not in text + # bedrock-runtime is the default; a mantle_* row needs a recorded selector + # and the OpenRouter same-model route has a runtime spelling. + assert "mantle_selector" in text, ( + f"{f}: design must require bedrock_models[].mantle_selector on mantle_* rows" + ) + assert "same_model_runtime" in text, ( + f"{f}: openrouter_path vocabulary must include same_model_runtime" + ) def test_generate_artifacts_ai_dispatches_runtime_openai_cris() -> None: @@ -122,6 +130,11 @@ def test_schema_design_aws_ai_vocabulary_includes_runtime_openai_cris() -> None: assert "mantle-only, no CRIS" not in normalized assert "**bare** proprietary `openai.gpt-*`" in normalized assert "us.openai.gpt-5.5" in normalized or "GPT-5.5 / GPT-5.4 runtime support" in normalized + # Runtime-default vocabulary: Anthropic SDK on bedrock-runtime, the runtime + # OpenRouter route, and the selector every mantle_* row must carry. + assert "runtime_messages" in text + assert "same_model_runtime" in text + assert "mantle_selector" in text # --- Finding 3: Astra Invoke-support correction --- diff --git a/plugins/aws-startup-advisor/tests/test_gpt6_round4_findings.py b/plugins/aws-startup-advisor/tests/test_gpt6_round4_findings.py index 46e2ac3a8..4481677d1 100644 --- a/plugins/aws-startup-advisor/tests/test_gpt6_round4_findings.py +++ b/plugins/aws-startup-advisor/tests/test_gpt6_round4_findings.py @@ -93,6 +93,9 @@ def test_azure_design_assert_scopes_gpt_ban_to_bare_ids_and_carves_out_cris() -> assert "GPT-5.5/GPT-5.4 us./global. profiles" in text assert "no proprietary GPT-5.5 or GPT-5.4 model ID is paired" not in text assert "no proprietary openai.gpt-* model ID is paired" not in text + assert "records migration_path runtime_openai_cris by default" in text, ( + "design.md _assert must state the bedrock-runtime default for same-model GPT rows" + ) def test_azure_generate_assert_scopes_gpt_ban_to_bare_ids_and_carves_out_cris() -> None: @@ -105,6 +108,9 @@ def test_azure_generate_assert_scopes_gpt_ban_to_bare_ids_and_carves_out_cris() assert "GPT-5.5/GPT-5.4 us./global. profiles" in text assert "no proprietary GPT-5.5 or GPT-5.4 model ID is paired" not in text assert "no proprietary openai.gpt-* model ID is paired" not in text + assert "records migration_path runtime_openai_cris by default" in text, ( + "generate.md _assert must state the bedrock-runtime default for same-model GPT rows" + ) # --- Finding 0.2: GCP generate-ai.md Part 1 routing --- From 5bdbc2feab86b8640aa16811ba66f608cd68740c Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 7 Nov 2026 09:42:24 -0800 Subject: [PATCH 09/41] feat(aws-startup-advisor): default Anthropic SDK migrations to the bedrock-runtime Messages endpoint in the shared guide and design/generate phases The canonical ai-anthropic-to-bedrock.md now keeps anthropic.Anthropic and repoints it at https://bedrock-runtime.\{region\}.amazonaws.com/anthropic with a string Bedrock token minted per client (provide_token), records migration_path "runtime_messages" + required_client_api "messages", moves the boto3 Converse rewrite under an explicit-alternative section, and gates Mantle Messages behind a mantle_selector. gcp/azure design-ai.md Anthropic lines, gcp/azure generate-artifacts-ai.md Step 0/Step 1, the estimate route lists, skills/shared/ai/README.md, gcp SKILL.md and the behavior-delta Anthropic reference carry the same wording; vendored copies re-synced. Public basis: inference-messages-api.html and inference-profiles-prereq.html (read 2026-11-07). Verification (plugins/aws-startup-advisor): - python3 tools/sync-vendored.py --check: PASS, 64 files byte-identical - uv run --no-project --with pytest python -m pytest tests -q: 1407 passed, 222 subtests - uv run --project skills/llm-to-bedrock/scripts --with pytest python -m pytest skills/llm-to-bedrock/scripts -q: 418 passed - mise run lint:md: 823 files, 0 errors; mise run lint:skill-refs: 1514 checked, 0 missing; mise run lint:vendored-parity: PASS 64 - grep -rn 'Converse client swap' skills agents | grep -v vendored: no matches --- .../references/phases/design/design-ai.md | 9 +- .../phases/estimate/estimate-assemble.md | 5 +- .../phases/generate/generate-artifacts-ai.md | 10 +- .../vendored/ai/ai-anthropic-to-bedrock.md | 150 +++++++++++++----- .../skills/gcp-to-aws/SKILL.md | 2 +- .../references/phases/design/design-ai.md | 4 +- .../references/phases/estimate/estimate.md | 5 +- .../phases/generate/generate-artifacts-ai.md | 6 +- .../vendored/ai/ai-anthropic-to-bedrock.md | 150 +++++++++++++----- .../references/anthropic-to-bedrock.md | 5 +- .../skills/shared/ai/README.md | 2 +- .../shared/ai/ai-anthropic-to-bedrock.md | 150 +++++++++++++----- .../tests/test_endpoint_policy_contract.py | 26 +++ 13 files changed, 392 insertions(+), 132 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md index dd84ff7fe..0e11e259c 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md @@ -41,7 +41,9 @@ defaults (prefer managed Bedrock, no latency constraint, no budget cap). unless a `mantle_selector` from `references/vendored/ai/bedrock-endpoint-policy.md` is recorded. `azure_openai` routes IDENTICALLY to `openai`; the distinct label is kept only for report wording and provenance (§19.9(b)). - `"anthropic"` → load `references/vendored/ai/ai-anthropic-to-bedrock.md` (Anthropic SDK → - Bedrock Converse client swap). + Bedrock Runtime Messages (default) / Converse alternative: the Anthropic SDK stays and is + repointed at `https://bedrock-runtime.{region}.amazonaws.com/anthropic`; record + `migration_path: "runtime_messages"` with `required_client_api: "messages"` by default). - `"both"` → load `references/vendored/ai/ai-openai-to-bedrock.md` and, if an Anthropic model is also present, `ai-anthropic-to-bedrock.md`. - `"other"` or absent → load `references/design-refs/ai.md` (traditional-ML rubric — Azure AI @@ -278,7 +280,10 @@ to the selected Bedrock client; do not claim that setting OPENAI_BASE_URL alone Keep `model_change: false` only when the exact underlying model is unchanged; opaque Azure deployment names need verified model identity, not a guessed GPT family. -Anthropic-SDK sources → boto3 Converse client swap. Record `code_migration.openrouter_path` +Anthropic-SDK sources → keep the Anthropic SDK on the `bedrock-runtime` Messages endpoint: +`migration_path: "runtime_messages"` with `required_client_api: "messages"` by default; +`"converse"` (boto3 rewrite) only on explicit request or for a Converse-only feature; +`"mantle_messages"` only with a `mantle_selector`. Record `code_migration.openrouter_path` when a router was detected (`same_model_runtime|same_model_mantle|direct|litellm|keep_openrouter`; `same_model_runtime` is the default same-model route, `same_model_mantle` needs a `mantle_selector`). diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate-assemble.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate-assemble.md index 2bb0b9dcd..0b5f7df2d 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate-assemble.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/estimate/estimate-assemble.md @@ -132,8 +132,9 @@ Step 3B does — and then show only the matching label: 2. **Otherwise** read `ai_architecture.code_migration.migration_path`: - `runtime_openai_cris` (Design's default) → the Bedrock runtime (CRIS) label above. - `mantle_openai_responses` / `mantle_openai_chat` (Design's spellings, selector-gated) or `mantle` → the Mantle label above. - - `converse`, `direct`, or `gpt-oss` → the adapter label above. - `converse` is a Converse client swap, not a Mantle script and not gateway config. + - `converse`, `direct`, `runtime_messages`, `mantle_messages`, or `gpt-oss` → the adapter label above. + `converse` is an explicit Converse rewrite, not a Mantle script and not gateway config; + the two Messages paths keep the Anthropic SDK inside the adapter. - `llm_router` / `api_gateway` / `voice_platform` / `framework` → the gateway label above. This bullet does not apply when step 1 already matched an OpenRouter subroute. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md index 279af5d92..3f6f17a68 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -29,7 +29,9 @@ Read `aws-design-ai.json → ai_architecture.code_migration.migration_path` and | ----------------------------------------------------------------- | ------------------------------------------------------------------------- | | `runtime_openai_cris` (default same-model path) | `ai-migration/migrate_to_mantle.sh` (CRIS variant — `bedrock-runtime` base URL + CRIS id + the row's `required_client_api`) — skip the provider adapter | | `mantle_openai_responses` / `mantle_openai_chat` (only with `mantle_selector`) | `ai-migration/migrate_to_mantle.sh` — **skip the provider adapter** | -| `direct` (or absent) or `converse` | `ai-migration/provider_adapter.{py,js,go}` — `converse` is a Converse client swap, not a Mantle script | +| `runtime_messages` (Anthropic source, default) | `ai-migration/provider_adapter.{py,js,go}` whose Bedrock class keeps the Anthropic SDK at `https://bedrock-runtime.{region}.amazonaws.com/anthropic` with a string Bedrock token (`required_client_api: messages`; `bedrock:InvokeModel*` IAM) — not a Mantle script, not a Converse rewrite | +| `mantle_messages` (Anthropic source, only with `mantle_selector`) | same adapter shape at `https://bedrock-mantle.{region}.api.aws/anthropic` with `bedrock-mantle:*` IAM | +| `direct` (or absent) or `converse` | `ai-migration/provider_adapter.{py,js,go}` — `converse` is an explicit boto3 Converse rewrite, not a Mantle script | | `gpt-oss` | `ai-migration/provider_adapter.{py,js,go}` targeting gpt-oss via Converse | | gateway (`llm_router`/`api_gateway`/`voice_platform`/`framework`) | `ai-migration/gateway_config.{yaml,py,json}` — skip the adapter | | agentic `migration_approach == harness` | `ai-migration/harness.json`, `ai-migration/deploy_harness.sh` | @@ -71,7 +73,11 @@ Adapt Azure-specific client/deployment/API-version settings to the Bedrock clien Feature-flagged on `AI_PROVIDER` (values `azure_openai` | `bedrock` | `shadow`; **default `azure_openai`**). Emits methods gated on `integration.capabilities_summary`: `text_generation`→ `generate`, `streaming`→`generate_stream`, `embeddings`→`embed`. Bedrock side is boto3 Converse -(`BEDROCK_INFERENCE_PROFILE_ARN` env, model-id fallback). **Azure source side:** the `azure_openai` +(`BEDROCK_INFERENCE_PROFILE_ARN` env, model-id fallback); for `runtime_messages` it is instead +`anthropic.Anthropic(base_url="https://bedrock-runtime.{region}.amazonaws.com/anthropic", +api_key=provide_token(region=region))` — a string token minted per client, client created and +closed per request boundary, `messages.create` / `messages.stream` unchanged, `global.`/Geo +Claude profile ID (`mantle_messages`: same shape at `https://bedrock-mantle.{region}.api.aws/anthropic`). **Azure source side:** the `azure_openai` branch calls the `AzureOpenAI` client (`azure_endpoint`, `api_version`) for Python / `@azure/openai` (or the `openai` SDK with `azure` config) for Node — NOT Vertex/`@google-cloud/vertexai`. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md index 6fa440b55..59d6a0796 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md @@ -1,16 +1,23 @@ # Anthropic SDK → Amazon Bedrock Migration -> Canonical Anthropic-SDK → Bedrock client-swap guide. Source-cloud agnostic: -> the mapping depends on the SDK the code uses, never on which cloud hosted it. -> Vendored into each consuming skill as +> Canonical Anthropic SDK → Bedrock Runtime Messages (default) / Converse +> alternative guide. Source-cloud agnostic: the mapping depends on the SDK the +> code uses, never on which cloud hosted it. Vendored into each consuming skill as > `references/vendored/ai/ai-anthropic-to-bedrock.md` and kept byte-identical by > `shared:check`; edit HERE, then run `shared:sync`. > > Loaded by `design-ai.md` for an Anthropic source, or when Claude Haiku 5.5 is a candidate or selected target. > For a non-Anthropic source, only the Haiku 5.5 target contract below applies; > use the source provider guide for SDK and request translation. Steps 1–7 and -> the Anthropic client-swap checklist apply only to Anthropic-source migrations. -> For an Anthropic source, the user is already on Claude via the Anthropic SDK. Migration is a client swap only. +> the Anthropic Messages checklist apply only to Anthropic-source migrations. +> For an Anthropic source, the user is already on Claude via the Anthropic SDK. +> The default migration keeps that SDK and points it at the `bedrock-runtime` +> Anthropic Messages endpoint (`migration_path: "runtime_messages"`, +> `required_client_api: "messages"`), per +> [Inference using Anthropic Messages API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html) +> (read 2026-11-07) and `references/vendored/ai/bedrock-endpoint-policy.md`. +> A Converse rewrite (`migration_path: "converse"`) is an explicit alternative; +> Mantle Messages (`migration_path: "mantle_messages"`) is selector-gated. > Mapping an older Claude version to a newer target is a model change. Validate > request controls and output behavior before cutover. @@ -35,57 +42,86 @@ Older Claude models — Claude 3.5 Haiku, Claude 3 Sonnet, Claude 3.5 Sonnet (v1 --- -## Step 2: Rewrite client +## Step 2: Repoint the client (Runtime Messages, default) -Before: `anthropic.Anthropic(api_key=...)` -After: `boto3.client("bedrock-runtime", region_name="us-east-1")` +Keep `anthropic.Anthropic`. Change only the base URL and the credential +([Inference using Anthropic Messages API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html), read 2026-11-07): -Remove ANTHROPIC_API_KEY. Use IAM role with bedrock:InvokeModel. +Before: `anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])` ---- +After: -## Step 3: Rewrite API calls +```python +from anthropic import Anthropic +from aws_bedrock_token_generator import provide_token + +region = "us-east-1" + +def handle_request(prompt: str) -> str: + # Mint one short-term Bedrock bearer token per client; the SDK accepts only a + # string credential here, so create the client per request boundary and close it. + with Anthropic( + base_url=f"https://bedrock-runtime.{region}.amazonaws.com/anthropic", + api_key=provide_token(region=region), + ) as client: + ... +``` -Before: `client.messages.create(model=..., max_tokens=1024, messages=[{"role": "user", "content": "Hello"}])` +- The base URL ends at `/anthropic`; the SDK appends `/v1/messages` itself and adds the + required `anthropic-version` header. +- `api_key` is a **string** Bedrock token from `aws_bedrock_token_generator.provide_token` + minted from the application's IAM credentials. Mint it per client and re-create the client at + each request boundary; do not cache one token for the process lifetime. +- A short-lived CLI or test may instead export `AWS_BEARER_TOKEN_BEDROCK` (a Bedrock API key) + and pass it as `api_key`; long-running services use `provide_token`. +- Remove `ANTHROPIC_API_KEY` from environment and secrets; the credential is now IAM-derived. -After: `client.converse(modelId="anthropic.claude-sonnet-5", messages=[{"role": "user", "content": [{"text": "Hello"}]}], inferenceConfig={"maxTokens": 1024})` +--- -Key differences: +## Step 3: API calls -- content is typed blocks [{"text": "..."}] not a plain string -- max_tokens moves to inferenceConfig.maxTokens -- Select text blocks by their `text` key; reasoning and tool blocks can appear first. +`client.messages.create(...)` is unchanged except for the model ID: use a supported `global.` +or Geo (`us.`/`eu.`/`au.`/`jp.`) inference-profile ID from Step 1 (for example +`global.anthropic.claude-sonnet-5`). Use the bare `anthropic.claude-*` ID only where the model +card documents an in-region runtime endpoint (today the `eu-west-2` Opus 5.5 route below). + +Response parsing keeps the native shape: select text blocks by `type == "text"`, never by +position, because thinking and tool-use blocks can appear first. ```python -text = "".join(block["text"] for block in response["output"]["message"]["content"] if "text" in block) -stop_reason = response.get("stopReason", "unknown") -if not text or stop_reason in ("max_tokens", "model_context_window_exceeded", "guardrail_intervened", "content_filtered", "refusal", "tool_use"): - raise ValueError(f"No complete text response (stopReason={stop_reason})") +text = "".join(block.text for block in response.content if block.type == "text") +if not text or response.stop_reason in ("max_tokens", "refusal", "tool_use"): + raise ValueError(f"No complete text response (stop_reason={response.stop_reason})") ``` -For tool loops, retain the original `response["output"]["message"]` in assistant history, -including signed reasoning blocks, and handle every tool request before continuing. -The extraction above is for calls that expect a final text answer. +For tool loops, retain the complete assistant `response.content` in history, including +signed thinking blocks, and handle every tool request before continuing. --- ## Step 4: System prompts -Before: `system="You are helpful"` as a string param -After: `system=[{"text": "You are helpful"}]` as a list of blocks +Unchanged: `system="You are helpful"` stays a string (or a list of native blocks) parameter. --- ## Step 5: Streaming -Before: `client.messages.stream()` context manager -After: `client.converse_stream()` — iterate `response["stream"]` for `contentBlockDelta` events +Unchanged: `with client.messages.stream(...) as stream:` and `stream.text_stream` / +`stream.get_final_message()` work as before. Keep the SDK accumulator so signed blocks stay +complete while only text deltas reach text consumers. --- ## Step 6: IAM permissions -Add bedrock:InvokeModel and bedrock:InvokeModelWithResponseStream on arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-* +Source: [Prerequisites for inference profiles](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-prereq.html) (read 2026-11-07). + +- `bedrock:InvokeModel` and `bedrock:InvokeModelWithResponseStream` on the inference-profile ARN + (`arn:aws:bedrock:{region}:{account}:inference-profile/global.anthropic.claude-sonnet-5`) **and** + on the `foundation-model/anthropic.claude-*` ARN in every destination Region the profile routes to. +- `bedrock:GetInferenceProfile` on the inference-profile ARN. +- No `bedrock-mantle:*` permission is involved on this lane. --- @@ -95,16 +131,52 @@ Request TPM increases via Service Quotas. Cross-region inference profiles (us.* --- -## Validation checklist - -- [ ] anthropic.Anthropic() replaced with boto3.client("bedrock-runtime") -- [ ] client.messages.create() replaced with client.converse() -- [ ] content blocks use typed format [{"text": "..."}] -- [ ] max_tokens moved to inferenceConfig.maxTokens -- [ ] Streaming uses converse_stream with contentBlockDelta events -- [ ] ANTHROPIC_API_KEY removed from environment/secrets -- [ ] IAM role has bedrock:InvokeModel permission -- [ ] Model IDs updated to Bedrock format (Claude 4.x recommended) +## Validation checklist (Runtime Messages lane) + +- [ ] `anthropic.Anthropic` retained; `base_url` is `https://bedrock-runtime.{region}.amazonaws.com/anthropic` (ends at `/anthropic`; the SDK sends `/anthropic/v1/messages`) +- [ ] `api_key` is a string Bedrock token (`provide_token(region=...)` per client, or `AWS_BEARER_TOKEN_BEDROCK` for a short CLI); `ANTHROPIC_API_KEY` removed from environment/secrets +- [ ] Client created per request boundary and closed (`with Anthropic(...) as client:`) +- [ ] `messages.create` / `messages.stream` calls unchanged apart from the model ID +- [ ] Model IDs are supported `global.`/Geo inference profiles (bare ID only on a documented in-region route) +- [ ] Text selected by `type == "text"`; signed thinking/tool blocks preserved in tool history +- [ ] IAM grants `bedrock:InvokeModel*` on the profile and destination foundation-model ARNs plus `bedrock:GetInferenceProfile` on the profile +- [ ] `aws-design-ai.json` row records `migration_path: "runtime_messages"` and `required_client_api: "messages"` + +## Converse alternative (explicit) + +Record `migration_path: "converse"` only when the user asks for a Bedrock-native request shape or +needs a Converse-only feature (Guardrails on Converse, application inference profiles, per-request +metadata). This is a rewrite, not a repoint: + +- Client: `anthropic.Anthropic(api_key=...)` → `boto3.client("bedrock-runtime", region_name="us-east-1")`. +- Calls: `client.messages.create(model=..., max_tokens=1024, messages=[{"role": "user", "content": "Hello"}])` + → `client.converse(modelId="anthropic.claude-sonnet-5", messages=[{"role": "user", "content": [{"text": "Hello"}]}], inferenceConfig={"maxTokens": 1024})`. + Content becomes typed blocks `[{"text": "..."}]`; `max_tokens` moves to `inferenceConfig.maxTokens`; + select text blocks by their `text` key because reasoning and tool blocks can appear first: + + ```python + text = "".join(block["text"] for block in response["output"]["message"]["content"] if "text" in block) + stop_reason = response.get("stopReason", "unknown") + if not text or stop_reason in ("max_tokens", "model_context_window_exceeded", "guardrail_intervened", "content_filtered", "refusal", "tool_use"): + raise ValueError(f"No complete text response (stopReason={stop_reason})") + ``` + + For tool loops, retain the original `response["output"]["message"]` in assistant history, + including signed reasoning blocks, and handle every tool request before continuing. +- System prompts: `system="You are helpful"` → `system=[{"text": "You are helpful"}]`. +- Streaming: `client.messages.stream()` → `client.converse_stream()`; iterate `response["stream"]` + for `contentBlockDelta` events. +- IAM: the same `bedrock:InvokeModel*` grants as Step 6; `ANTHROPIC_API_KEY` removed. + +## Mantle Messages (selector-gated) + +Record `migration_path: "mantle_messages"` only together with a `bedrock_models[].mantle_selector` +from `references/vendored/ai/bedrock-endpoint-policy.md` — normally `workspaces` (Anthropic-compatible +Workspaces exist only on `bedrock-mantle`). The client shape is the Step 2 repoint with +`base_url="https://bedrock-mantle.{region}.api.aws/anthropic"`, a Bedrock API key or token as the +string `api_key`, the bare `anthropic.claude-*` model ID in a Mantle-listed Region, and +`bedrock-mantle:*` IAM instead of `bedrock:InvokeModel*`. API continuity alone never selects Mantle; +the Runtime lane above already keeps the Anthropic SDK. ## Opus 5.5 target contract (verified 2026-09-24) diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md index 5b8cf0bef..ff7631cc0 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/SKILL.md @@ -369,7 +369,7 @@ gcp-to-aws/ ├── ai-migration-guardrails.md # shared constraints for every agentic migration path ├── bedrock-quotas.md # Bedrock TPM/RPM quota awareness, burndown rates, capacity planning ├── ai-openai-to-bedrock.md # OpenAI-protocol → Bedrock model selection - ├── ai-anthropic-to-bedrock.md # Anthropic SDK → Bedrock Converse client swap + ├── ai-anthropic-to-bedrock.md # Anthropic SDK → Bedrock Runtime Messages (default) / Converse alternative ├── design-ref-harness.md # AgentCore Harness design reference ├── design-ref-agentic-to-agentcore.md # Strands Agents + AgentCore Runtime design reference └── sdk-capability-map.json # SDK method → capability lookup (discover-app-code.md) diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index 1fb918c0d..b9ede2a5a 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -27,7 +27,7 @@ Read `$MIGRATION_DIR/preferences.json` → `ai_constraints` (if present). If abs `references/shared/openai-on-bedrock.md` (the fact base: model IDs, endpoint paths, region matrix, quotas, caching). OpenAI's proprietary GPT models run on Bedrock, so the source model is often itself the target — the mapping guide's Tier 0 path is the default, not a cross-family swap. -- `"anthropic"` → load `references/vendored/ai/ai-anthropic-to-bedrock.md` (Anthropic SDK → Bedrock Converse API client swap; do NOT use ai-openai-to-bedrock.md for Anthropic SDK users) +- `"anthropic"` → load `references/vendored/ai/ai-anthropic-to-bedrock.md` (Anthropic SDK → Bedrock Runtime Messages (default) / Converse alternative; the Anthropic SDK stays and is repointed at `https://bedrock-runtime.{region}.amazonaws.com/anthropic` — record `migration_path: "runtime_messages"` with `required_client_api: "messages"` by default, `"converse"` only on explicit request or for a Converse-only feature, `"mantle_messages"` only with a `mantle_selector`; do NOT use ai-openai-to-bedrock.md for Anthropic SDK users) - `"both"` → `both` never means exactly "Gemini+OpenAI" — per `discover-anthropic-api.md` Step 4.5, `both` is also reached when Anthropic usage joins a Gemini-or-OpenAI codebase, so it can mean "Gemini+Anthropic" @@ -395,7 +395,7 @@ For each detected `integration.pattern` and `ai_source`, generate before/after m | Direct SDK (OpenAI), same model | OpenAI | Selected Mantle API (selector-gated) | Only with `mantle_selector`: base URL → `.../openai/v1` on Mantle, Bedrock credential, `openai.gpt-*` model ID, `bedrock-mantle:*` IAM | | Direct SDK | Vertex AI | boto3 Converse API | `generate_content()` → `converse()` | | Direct SDK | OpenAI | boto3 Converse API | `completions.create()` → `converse()` (use only for a need no same-model path serves, e.g. Knowledge Bases) | -| Direct SDK | Anthropic | boto3 Converse API | `messages.create()` → `converse()` with Claude model ID on Bedrock | +| Direct SDK | Anthropic | Runtime Anthropic Messages (default) | Keep `anthropic.Anthropic`; base URL → `https://bedrock-runtime.{region}.amazonaws.com/anthropic`, string Bedrock token, `global.`/Geo Claude profile ID, `bedrock:InvokeModel*` IAM; `migration_path: "runtime_messages"` + `required_client_api: "messages"`. `"converse"` (boto3 `messages.create()` → `converse()`) only on explicit request / Converse-only feature; `"mantle_messages"` only with a `mantle_selector` | | LangChain | ChatVertexAI / ChatOpenAI | ChatBedrock | Swap import and model_id | | LlamaIndex | Vertex / OpenAI LLM | BedrockConverse | Swap import | | LLM Router (LiteLLM) | Any | Config change | `model="bedrock/"` (1 line) | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md index 1500313e5..1416fb8f8 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/estimate/estimate.md @@ -335,8 +335,9 @@ in this order — the OpenRouter subroute wins over `migration_path`, because `"converse"`; accept every Mantle spelling: - `"runtime_openai_cris"` → the Bedrock runtime (CRIS) label above. - `"mantle"`, `"mantle_openai_responses"` or `"mantle_openai_chat"` → the Mantle label above. - - `"converse"`, `"direct"`, or `"gpt-oss"` → the adapter label above. - `"converse"` is a Converse client swap, not a Mantle script and not gateway config. + - `"converse"`, `"direct"`, `"runtime_messages"`, `"mantle_messages"`, or `"gpt-oss"` → the adapter label above. + `"converse"` is an explicit Converse rewrite, not a Mantle script and not gateway config; + the two Messages paths keep the Anthropic SDK inside the adapter. - `"llm_router"`, `"api_gateway"`, `"voice_platform"`, or `"framework"` → the gateway label above. This bullet does not apply when step 1 already matched an OpenRouter subroute. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md index d9bdc22f7..ab7569d29 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -59,7 +59,9 @@ Check `preferences.json` → `ai_constraints.ai_framework.value` and `aws-design (Step 2). Skip provider adapter (Step 1). - `migration_path` starts with `"mantle"` (`"mantle"`, `"mantle_openai_chat"`, `"mantle_openai_responses"`) → Generate Mantle setup (Step 1M) + setup (Step 3) + test harness (Step 2). Skip provider adapter (Step 1). Design records `"mantle_openai_responses"` or `"mantle_openai_chat"` only together with `bedrock_models[].mantle_selector` (see `references/vendored/ai/bedrock-endpoint-policy.md`); echo that selector in the generated script's header so the Mantle choice stays traceable. Treat the `"mantle_openai_responses"` spelling as this same Mantle path. - `migration_path` = `"gpt-oss"` → Generate provider adapter (Step 1) targeting the gpt-oss model on Bedrock via the Converse API + setup (Step 3) + test harness (Step 2). Use the Bedrock model ID from `aws-design-ai.json` → `ai_architecture.bedrock_models[].aws_model_id` (the gpt-oss model ID). Do NOT generate a Mantle script — gpt-oss uses the Converse API directly. -- `"converse"`, `"direct"`, or absent → Generate provider adapter (Step 1) + setup (Step 3) + test harness (Step 2). `"converse"` is a Converse client swap (Design records it when a proprietary model cannot stay on Mantle); do NOT generate a Mantle script for it. +- `migration_path` = `"runtime_messages"` (Anthropic source; Design's default for an Anthropic SDK app) → Generate provider adapter (Step 1) whose Bedrock provider class keeps the Anthropic SDK at the runtime base URL `https://bedrock-runtime.{region}.amazonaws.com/anthropic` with a string Bedrock token (`provide_token(region=...)` minted per client) and the row's `global.`/Geo Claude profile ID + setup (Step 3, `bedrock:InvokeModel*` and `bedrock:GetInferenceProfile` IAM) + test harness (Step 2). `required_client_api` is `"messages"`; do NOT generate a Mantle script and do NOT rewrite to Converse. +- `migration_path` = `"mantle_messages"` (Anthropic source, only with `bedrock_models[].mantle_selector`, normally `workspaces`) → the same adapter shape at the Mantle base URL `https://bedrock-mantle.{region}.api.aws/anthropic` with the bare `anthropic.claude-*` ID and `bedrock-mantle:*` IAM; echo the selector in the adapter header. +- `"converse"`, `"direct"`, or absent → Generate provider adapter (Step 1) + setup (Step 3) + test harness (Step 2). `"converse"` is an explicit boto3 Converse rewrite (Design records it on request or for a Converse-only feature); do NOT generate a Mantle script for it. - `"llm_router"`, `"api_gateway"`, `"voice_platform"`, or `"framework"` → Skip Step 1, generate gateway config (Step 3B) instead. Step 3B's OpenRouter subroute can still replace that gateway config with Step 1M (`same_model_runtime` → the CRIS variant, `same_model_mantle` → the Mantle variant) or the provider adapter (`direct`); do not decide those from `migration_path` alone. For `runtime_openai_cris`, read the matched `bedrock_models[].required_client_api` — the API the @@ -124,7 +126,7 @@ Generate `ai-migration/provider_adapter.{py,js,go}` — an abstraction layer tha - `streaming: true` → `generate_stream(prompt) → Iterator[str]` - `embeddings: true` → `embed(text) → list[float]` - **Source provider class**: Use SDK imports from `ai-workload-profile.json` → `integration.sdk_imports`. Use model IDs from `ai-workload-profile.json` → `models[].model_id`. -- **Bedrock provider class**: For a Converse plan, use `boto3` (`converse` for generate, `converse_stream` for streaming, `invoke_model` for separately selected Titan embeddings). For `runtime_openai_cris`, preserve the runtime API selected in the design: Converse uses this adapter shape; an OpenAI-compatible Chat/Responses selection uses the OpenAI SDK at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` with Bedrock credentials and the exact CRIS ID. Never send a bare Astra ID to runtime or replace a validated `global.` profile with `us.`. Use an application inference profile only when the selected model/API supports it; otherwise retain `aws-design-ai.json` → `ai_architecture.bedrock_models[].aws_model_id`. Use region from `preferences.json` → `design_constraints.target_region`. +- **Bedrock provider class**: For a Converse plan, use `boto3` (`converse` for generate, `converse_stream` for streaming, `invoke_model` for separately selected Titan embeddings). For `runtime_messages`, keep `anthropic.Anthropic(base_url="https://bedrock-runtime.{region}.amazonaws.com/anthropic", api_key=provide_token(region=region))` — a string token minted per client, the client created and closed per request boundary — with `messages.create` / `messages.stream` unchanged and the `global.`/Geo profile ID from the design row (`mantle_messages`: same shape at `https://bedrock-mantle.{region}.api.aws/anthropic`). For `runtime_openai_cris`, preserve the runtime API selected in the design: Converse uses this adapter shape; an OpenAI-compatible Chat/Responses selection uses the OpenAI SDK at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` with Bedrock credentials and the exact CRIS ID. Never send a bare Astra ID to runtime or replace a validated `global.` profile with `us.`. Use an application inference profile only when the selected model/API supports it; otherwise retain `aws-design-ai.json` → `ai_architecture.bedrock_models[].aws_model_id`. Use region from `preferences.json` → `design_constraints.target_region`. ```python # Use inference profile ARN for cost attribution (preferred) diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md index 6fa440b55..59d6a0796 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md @@ -1,16 +1,23 @@ # Anthropic SDK → Amazon Bedrock Migration -> Canonical Anthropic-SDK → Bedrock client-swap guide. Source-cloud agnostic: -> the mapping depends on the SDK the code uses, never on which cloud hosted it. -> Vendored into each consuming skill as +> Canonical Anthropic SDK → Bedrock Runtime Messages (default) / Converse +> alternative guide. Source-cloud agnostic: the mapping depends on the SDK the +> code uses, never on which cloud hosted it. Vendored into each consuming skill as > `references/vendored/ai/ai-anthropic-to-bedrock.md` and kept byte-identical by > `shared:check`; edit HERE, then run `shared:sync`. > > Loaded by `design-ai.md` for an Anthropic source, or when Claude Haiku 5.5 is a candidate or selected target. > For a non-Anthropic source, only the Haiku 5.5 target contract below applies; > use the source provider guide for SDK and request translation. Steps 1–7 and -> the Anthropic client-swap checklist apply only to Anthropic-source migrations. -> For an Anthropic source, the user is already on Claude via the Anthropic SDK. Migration is a client swap only. +> the Anthropic Messages checklist apply only to Anthropic-source migrations. +> For an Anthropic source, the user is already on Claude via the Anthropic SDK. +> The default migration keeps that SDK and points it at the `bedrock-runtime` +> Anthropic Messages endpoint (`migration_path: "runtime_messages"`, +> `required_client_api: "messages"`), per +> [Inference using Anthropic Messages API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html) +> (read 2026-11-07) and `references/vendored/ai/bedrock-endpoint-policy.md`. +> A Converse rewrite (`migration_path: "converse"`) is an explicit alternative; +> Mantle Messages (`migration_path: "mantle_messages"`) is selector-gated. > Mapping an older Claude version to a newer target is a model change. Validate > request controls and output behavior before cutover. @@ -35,57 +42,86 @@ Older Claude models — Claude 3.5 Haiku, Claude 3 Sonnet, Claude 3.5 Sonnet (v1 --- -## Step 2: Rewrite client +## Step 2: Repoint the client (Runtime Messages, default) -Before: `anthropic.Anthropic(api_key=...)` -After: `boto3.client("bedrock-runtime", region_name="us-east-1")` +Keep `anthropic.Anthropic`. Change only the base URL and the credential +([Inference using Anthropic Messages API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html), read 2026-11-07): -Remove ANTHROPIC_API_KEY. Use IAM role with bedrock:InvokeModel. +Before: `anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])` ---- +After: -## Step 3: Rewrite API calls +```python +from anthropic import Anthropic +from aws_bedrock_token_generator import provide_token + +region = "us-east-1" + +def handle_request(prompt: str) -> str: + # Mint one short-term Bedrock bearer token per client; the SDK accepts only a + # string credential here, so create the client per request boundary and close it. + with Anthropic( + base_url=f"https://bedrock-runtime.{region}.amazonaws.com/anthropic", + api_key=provide_token(region=region), + ) as client: + ... +``` -Before: `client.messages.create(model=..., max_tokens=1024, messages=[{"role": "user", "content": "Hello"}])` +- The base URL ends at `/anthropic`; the SDK appends `/v1/messages` itself and adds the + required `anthropic-version` header. +- `api_key` is a **string** Bedrock token from `aws_bedrock_token_generator.provide_token` + minted from the application's IAM credentials. Mint it per client and re-create the client at + each request boundary; do not cache one token for the process lifetime. +- A short-lived CLI or test may instead export `AWS_BEARER_TOKEN_BEDROCK` (a Bedrock API key) + and pass it as `api_key`; long-running services use `provide_token`. +- Remove `ANTHROPIC_API_KEY` from environment and secrets; the credential is now IAM-derived. -After: `client.converse(modelId="anthropic.claude-sonnet-5", messages=[{"role": "user", "content": [{"text": "Hello"}]}], inferenceConfig={"maxTokens": 1024})` +--- -Key differences: +## Step 3: API calls -- content is typed blocks [{"text": "..."}] not a plain string -- max_tokens moves to inferenceConfig.maxTokens -- Select text blocks by their `text` key; reasoning and tool blocks can appear first. +`client.messages.create(...)` is unchanged except for the model ID: use a supported `global.` +or Geo (`us.`/`eu.`/`au.`/`jp.`) inference-profile ID from Step 1 (for example +`global.anthropic.claude-sonnet-5`). Use the bare `anthropic.claude-*` ID only where the model +card documents an in-region runtime endpoint (today the `eu-west-2` Opus 5.5 route below). + +Response parsing keeps the native shape: select text blocks by `type == "text"`, never by +position, because thinking and tool-use blocks can appear first. ```python -text = "".join(block["text"] for block in response["output"]["message"]["content"] if "text" in block) -stop_reason = response.get("stopReason", "unknown") -if not text or stop_reason in ("max_tokens", "model_context_window_exceeded", "guardrail_intervened", "content_filtered", "refusal", "tool_use"): - raise ValueError(f"No complete text response (stopReason={stop_reason})") +text = "".join(block.text for block in response.content if block.type == "text") +if not text or response.stop_reason in ("max_tokens", "refusal", "tool_use"): + raise ValueError(f"No complete text response (stop_reason={response.stop_reason})") ``` -For tool loops, retain the original `response["output"]["message"]` in assistant history, -including signed reasoning blocks, and handle every tool request before continuing. -The extraction above is for calls that expect a final text answer. +For tool loops, retain the complete assistant `response.content` in history, including +signed thinking blocks, and handle every tool request before continuing. --- ## Step 4: System prompts -Before: `system="You are helpful"` as a string param -After: `system=[{"text": "You are helpful"}]` as a list of blocks +Unchanged: `system="You are helpful"` stays a string (or a list of native blocks) parameter. --- ## Step 5: Streaming -Before: `client.messages.stream()` context manager -After: `client.converse_stream()` — iterate `response["stream"]` for `contentBlockDelta` events +Unchanged: `with client.messages.stream(...) as stream:` and `stream.text_stream` / +`stream.get_final_message()` work as before. Keep the SDK accumulator so signed blocks stay +complete while only text deltas reach text consumers. --- ## Step 6: IAM permissions -Add bedrock:InvokeModel and bedrock:InvokeModelWithResponseStream on arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-* +Source: [Prerequisites for inference profiles](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-prereq.html) (read 2026-11-07). + +- `bedrock:InvokeModel` and `bedrock:InvokeModelWithResponseStream` on the inference-profile ARN + (`arn:aws:bedrock:{region}:{account}:inference-profile/global.anthropic.claude-sonnet-5`) **and** + on the `foundation-model/anthropic.claude-*` ARN in every destination Region the profile routes to. +- `bedrock:GetInferenceProfile` on the inference-profile ARN. +- No `bedrock-mantle:*` permission is involved on this lane. --- @@ -95,16 +131,52 @@ Request TPM increases via Service Quotas. Cross-region inference profiles (us.* --- -## Validation checklist - -- [ ] anthropic.Anthropic() replaced with boto3.client("bedrock-runtime") -- [ ] client.messages.create() replaced with client.converse() -- [ ] content blocks use typed format [{"text": "..."}] -- [ ] max_tokens moved to inferenceConfig.maxTokens -- [ ] Streaming uses converse_stream with contentBlockDelta events -- [ ] ANTHROPIC_API_KEY removed from environment/secrets -- [ ] IAM role has bedrock:InvokeModel permission -- [ ] Model IDs updated to Bedrock format (Claude 4.x recommended) +## Validation checklist (Runtime Messages lane) + +- [ ] `anthropic.Anthropic` retained; `base_url` is `https://bedrock-runtime.{region}.amazonaws.com/anthropic` (ends at `/anthropic`; the SDK sends `/anthropic/v1/messages`) +- [ ] `api_key` is a string Bedrock token (`provide_token(region=...)` per client, or `AWS_BEARER_TOKEN_BEDROCK` for a short CLI); `ANTHROPIC_API_KEY` removed from environment/secrets +- [ ] Client created per request boundary and closed (`with Anthropic(...) as client:`) +- [ ] `messages.create` / `messages.stream` calls unchanged apart from the model ID +- [ ] Model IDs are supported `global.`/Geo inference profiles (bare ID only on a documented in-region route) +- [ ] Text selected by `type == "text"`; signed thinking/tool blocks preserved in tool history +- [ ] IAM grants `bedrock:InvokeModel*` on the profile and destination foundation-model ARNs plus `bedrock:GetInferenceProfile` on the profile +- [ ] `aws-design-ai.json` row records `migration_path: "runtime_messages"` and `required_client_api: "messages"` + +## Converse alternative (explicit) + +Record `migration_path: "converse"` only when the user asks for a Bedrock-native request shape or +needs a Converse-only feature (Guardrails on Converse, application inference profiles, per-request +metadata). This is a rewrite, not a repoint: + +- Client: `anthropic.Anthropic(api_key=...)` → `boto3.client("bedrock-runtime", region_name="us-east-1")`. +- Calls: `client.messages.create(model=..., max_tokens=1024, messages=[{"role": "user", "content": "Hello"}])` + → `client.converse(modelId="anthropic.claude-sonnet-5", messages=[{"role": "user", "content": [{"text": "Hello"}]}], inferenceConfig={"maxTokens": 1024})`. + Content becomes typed blocks `[{"text": "..."}]`; `max_tokens` moves to `inferenceConfig.maxTokens`; + select text blocks by their `text` key because reasoning and tool blocks can appear first: + + ```python + text = "".join(block["text"] for block in response["output"]["message"]["content"] if "text" in block) + stop_reason = response.get("stopReason", "unknown") + if not text or stop_reason in ("max_tokens", "model_context_window_exceeded", "guardrail_intervened", "content_filtered", "refusal", "tool_use"): + raise ValueError(f"No complete text response (stopReason={stop_reason})") + ``` + + For tool loops, retain the original `response["output"]["message"]` in assistant history, + including signed reasoning blocks, and handle every tool request before continuing. +- System prompts: `system="You are helpful"` → `system=[{"text": "You are helpful"}]`. +- Streaming: `client.messages.stream()` → `client.converse_stream()`; iterate `response["stream"]` + for `contentBlockDelta` events. +- IAM: the same `bedrock:InvokeModel*` grants as Step 6; `ANTHROPIC_API_KEY` removed. + +## Mantle Messages (selector-gated) + +Record `migration_path: "mantle_messages"` only together with a `bedrock_models[].mantle_selector` +from `references/vendored/ai/bedrock-endpoint-policy.md` — normally `workspaces` (Anthropic-compatible +Workspaces exist only on `bedrock-mantle`). The client shape is the Step 2 repoint with +`base_url="https://bedrock-mantle.{region}.api.aws/anthropic"`, a Bedrock API key or token as the +string `api_key`, the bare `anthropic.claude-*` model ID in a Mantle-listed Region, and +`bedrock-mantle:*` IAM instead of `bedrock:InvokeModel*`. API continuity alone never selects Mantle; +the Runtime lane above already keeps the Anthropic SDK. ## Opus 5.5 target contract (verified 2026-09-24) diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/behavior-delta-detection/references/anthropic-to-bedrock.md b/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/behavior-delta-detection/references/anthropic-to-bedrock.md index 3b6299f24..6519fb708 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/behavior-delta-detection/references/anthropic-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/behavior-delta-detection/references/anthropic-to-bedrock.md @@ -59,7 +59,10 @@ rg -n 'tool_choice|toolChoice|prefill|json_schema|response_format|role.*assistan ## typed-response-and-history For every path, read visible text by block type, never by first-block position. -Native Messages uses `type: text`; Converse uses text-key blocks. Preserve the +Native Messages uses `type: text`; Converse uses text-key blocks. On the +`runtime_messages` and `mantle_messages` lanes the native `type: text` block shape +and the message history are preserved unchanged; only the explicit Converse +alternative reshapes them. Preserve the complete assistant response, including reasoning/signatures and tool-use blocks, when continuing tool history. Streaming must preserve complete signed blocks using the SDK's accumulator while emitting only text deltas to text consumers. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/README.md b/plugins/aws-startup-advisor/skills/shared/ai/README.md index 2113c122e..89278875b 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/README.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/README.md @@ -15,7 +15,7 @@ before opening a pull request. | `bedrock-endpoint-policy.md` | `bedrock-runtime` default / `bedrock-mantle` selector rule, `mantle_selector` enum, public endpoint matrix | | `bedrock-quotas.md` | TPM/RPM quota-risk assessment (works without an AWS account) | | `ai-openai-to-bedrock.md` | OpenAI-protocol (incl. Azure OpenAI) → Bedrock model selection | -| `ai-anthropic-to-bedrock.md` | Anthropic SDK → Bedrock Converse client swap | +| `ai-anthropic-to-bedrock.md` | Anthropic SDK → Bedrock Runtime Messages (default) / Converse alternative | | `ai-gemini-to-bedrock.md` | Gemini (incl. Vertex AI) → Bedrock model selection | | `ai-openrouter-to-bedrock.md` | OpenRouter gateway → per-upstream Bedrock routing | | `design-ref-harness.md` | AgentCore Harness design reference | diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-anthropic-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-anthropic-to-bedrock.md index 6fa440b55..59d6a0796 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-anthropic-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-anthropic-to-bedrock.md @@ -1,16 +1,23 @@ # Anthropic SDK → Amazon Bedrock Migration -> Canonical Anthropic-SDK → Bedrock client-swap guide. Source-cloud agnostic: -> the mapping depends on the SDK the code uses, never on which cloud hosted it. -> Vendored into each consuming skill as +> Canonical Anthropic SDK → Bedrock Runtime Messages (default) / Converse +> alternative guide. Source-cloud agnostic: the mapping depends on the SDK the +> code uses, never on which cloud hosted it. Vendored into each consuming skill as > `references/vendored/ai/ai-anthropic-to-bedrock.md` and kept byte-identical by > `shared:check`; edit HERE, then run `shared:sync`. > > Loaded by `design-ai.md` for an Anthropic source, or when Claude Haiku 5.5 is a candidate or selected target. > For a non-Anthropic source, only the Haiku 5.5 target contract below applies; > use the source provider guide for SDK and request translation. Steps 1–7 and -> the Anthropic client-swap checklist apply only to Anthropic-source migrations. -> For an Anthropic source, the user is already on Claude via the Anthropic SDK. Migration is a client swap only. +> the Anthropic Messages checklist apply only to Anthropic-source migrations. +> For an Anthropic source, the user is already on Claude via the Anthropic SDK. +> The default migration keeps that SDK and points it at the `bedrock-runtime` +> Anthropic Messages endpoint (`migration_path: "runtime_messages"`, +> `required_client_api: "messages"`), per +> [Inference using Anthropic Messages API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html) +> (read 2026-11-07) and `references/vendored/ai/bedrock-endpoint-policy.md`. +> A Converse rewrite (`migration_path: "converse"`) is an explicit alternative; +> Mantle Messages (`migration_path: "mantle_messages"`) is selector-gated. > Mapping an older Claude version to a newer target is a model change. Validate > request controls and output behavior before cutover. @@ -35,57 +42,86 @@ Older Claude models — Claude 3.5 Haiku, Claude 3 Sonnet, Claude 3.5 Sonnet (v1 --- -## Step 2: Rewrite client +## Step 2: Repoint the client (Runtime Messages, default) -Before: `anthropic.Anthropic(api_key=...)` -After: `boto3.client("bedrock-runtime", region_name="us-east-1")` +Keep `anthropic.Anthropic`. Change only the base URL and the credential +([Inference using Anthropic Messages API](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html), read 2026-11-07): -Remove ANTHROPIC_API_KEY. Use IAM role with bedrock:InvokeModel. +Before: `anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])` ---- +After: -## Step 3: Rewrite API calls +```python +from anthropic import Anthropic +from aws_bedrock_token_generator import provide_token + +region = "us-east-1" + +def handle_request(prompt: str) -> str: + # Mint one short-term Bedrock bearer token per client; the SDK accepts only a + # string credential here, so create the client per request boundary and close it. + with Anthropic( + base_url=f"https://bedrock-runtime.{region}.amazonaws.com/anthropic", + api_key=provide_token(region=region), + ) as client: + ... +``` -Before: `client.messages.create(model=..., max_tokens=1024, messages=[{"role": "user", "content": "Hello"}])` +- The base URL ends at `/anthropic`; the SDK appends `/v1/messages` itself and adds the + required `anthropic-version` header. +- `api_key` is a **string** Bedrock token from `aws_bedrock_token_generator.provide_token` + minted from the application's IAM credentials. Mint it per client and re-create the client at + each request boundary; do not cache one token for the process lifetime. +- A short-lived CLI or test may instead export `AWS_BEARER_TOKEN_BEDROCK` (a Bedrock API key) + and pass it as `api_key`; long-running services use `provide_token`. +- Remove `ANTHROPIC_API_KEY` from environment and secrets; the credential is now IAM-derived. -After: `client.converse(modelId="anthropic.claude-sonnet-5", messages=[{"role": "user", "content": [{"text": "Hello"}]}], inferenceConfig={"maxTokens": 1024})` +--- -Key differences: +## Step 3: API calls -- content is typed blocks [{"text": "..."}] not a plain string -- max_tokens moves to inferenceConfig.maxTokens -- Select text blocks by their `text` key; reasoning and tool blocks can appear first. +`client.messages.create(...)` is unchanged except for the model ID: use a supported `global.` +or Geo (`us.`/`eu.`/`au.`/`jp.`) inference-profile ID from Step 1 (for example +`global.anthropic.claude-sonnet-5`). Use the bare `anthropic.claude-*` ID only where the model +card documents an in-region runtime endpoint (today the `eu-west-2` Opus 5.5 route below). + +Response parsing keeps the native shape: select text blocks by `type == "text"`, never by +position, because thinking and tool-use blocks can appear first. ```python -text = "".join(block["text"] for block in response["output"]["message"]["content"] if "text" in block) -stop_reason = response.get("stopReason", "unknown") -if not text or stop_reason in ("max_tokens", "model_context_window_exceeded", "guardrail_intervened", "content_filtered", "refusal", "tool_use"): - raise ValueError(f"No complete text response (stopReason={stop_reason})") +text = "".join(block.text for block in response.content if block.type == "text") +if not text or response.stop_reason in ("max_tokens", "refusal", "tool_use"): + raise ValueError(f"No complete text response (stop_reason={response.stop_reason})") ``` -For tool loops, retain the original `response["output"]["message"]` in assistant history, -including signed reasoning blocks, and handle every tool request before continuing. -The extraction above is for calls that expect a final text answer. +For tool loops, retain the complete assistant `response.content` in history, including +signed thinking blocks, and handle every tool request before continuing. --- ## Step 4: System prompts -Before: `system="You are helpful"` as a string param -After: `system=[{"text": "You are helpful"}]` as a list of blocks +Unchanged: `system="You are helpful"` stays a string (or a list of native blocks) parameter. --- ## Step 5: Streaming -Before: `client.messages.stream()` context manager -After: `client.converse_stream()` — iterate `response["stream"]` for `contentBlockDelta` events +Unchanged: `with client.messages.stream(...) as stream:` and `stream.text_stream` / +`stream.get_final_message()` work as before. Keep the SDK accumulator so signed blocks stay +complete while only text deltas reach text consumers. --- ## Step 6: IAM permissions -Add bedrock:InvokeModel and bedrock:InvokeModelWithResponseStream on arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-* +Source: [Prerequisites for inference profiles](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-prereq.html) (read 2026-11-07). + +- `bedrock:InvokeModel` and `bedrock:InvokeModelWithResponseStream` on the inference-profile ARN + (`arn:aws:bedrock:{region}:{account}:inference-profile/global.anthropic.claude-sonnet-5`) **and** + on the `foundation-model/anthropic.claude-*` ARN in every destination Region the profile routes to. +- `bedrock:GetInferenceProfile` on the inference-profile ARN. +- No `bedrock-mantle:*` permission is involved on this lane. --- @@ -95,16 +131,52 @@ Request TPM increases via Service Quotas. Cross-region inference profiles (us.* --- -## Validation checklist - -- [ ] anthropic.Anthropic() replaced with boto3.client("bedrock-runtime") -- [ ] client.messages.create() replaced with client.converse() -- [ ] content blocks use typed format [{"text": "..."}] -- [ ] max_tokens moved to inferenceConfig.maxTokens -- [ ] Streaming uses converse_stream with contentBlockDelta events -- [ ] ANTHROPIC_API_KEY removed from environment/secrets -- [ ] IAM role has bedrock:InvokeModel permission -- [ ] Model IDs updated to Bedrock format (Claude 4.x recommended) +## Validation checklist (Runtime Messages lane) + +- [ ] `anthropic.Anthropic` retained; `base_url` is `https://bedrock-runtime.{region}.amazonaws.com/anthropic` (ends at `/anthropic`; the SDK sends `/anthropic/v1/messages`) +- [ ] `api_key` is a string Bedrock token (`provide_token(region=...)` per client, or `AWS_BEARER_TOKEN_BEDROCK` for a short CLI); `ANTHROPIC_API_KEY` removed from environment/secrets +- [ ] Client created per request boundary and closed (`with Anthropic(...) as client:`) +- [ ] `messages.create` / `messages.stream` calls unchanged apart from the model ID +- [ ] Model IDs are supported `global.`/Geo inference profiles (bare ID only on a documented in-region route) +- [ ] Text selected by `type == "text"`; signed thinking/tool blocks preserved in tool history +- [ ] IAM grants `bedrock:InvokeModel*` on the profile and destination foundation-model ARNs plus `bedrock:GetInferenceProfile` on the profile +- [ ] `aws-design-ai.json` row records `migration_path: "runtime_messages"` and `required_client_api: "messages"` + +## Converse alternative (explicit) + +Record `migration_path: "converse"` only when the user asks for a Bedrock-native request shape or +needs a Converse-only feature (Guardrails on Converse, application inference profiles, per-request +metadata). This is a rewrite, not a repoint: + +- Client: `anthropic.Anthropic(api_key=...)` → `boto3.client("bedrock-runtime", region_name="us-east-1")`. +- Calls: `client.messages.create(model=..., max_tokens=1024, messages=[{"role": "user", "content": "Hello"}])` + → `client.converse(modelId="anthropic.claude-sonnet-5", messages=[{"role": "user", "content": [{"text": "Hello"}]}], inferenceConfig={"maxTokens": 1024})`. + Content becomes typed blocks `[{"text": "..."}]`; `max_tokens` moves to `inferenceConfig.maxTokens`; + select text blocks by their `text` key because reasoning and tool blocks can appear first: + + ```python + text = "".join(block["text"] for block in response["output"]["message"]["content"] if "text" in block) + stop_reason = response.get("stopReason", "unknown") + if not text or stop_reason in ("max_tokens", "model_context_window_exceeded", "guardrail_intervened", "content_filtered", "refusal", "tool_use"): + raise ValueError(f"No complete text response (stopReason={stop_reason})") + ``` + + For tool loops, retain the original `response["output"]["message"]` in assistant history, + including signed reasoning blocks, and handle every tool request before continuing. +- System prompts: `system="You are helpful"` → `system=[{"text": "You are helpful"}]`. +- Streaming: `client.messages.stream()` → `client.converse_stream()`; iterate `response["stream"]` + for `contentBlockDelta` events. +- IAM: the same `bedrock:InvokeModel*` grants as Step 6; `ANTHROPIC_API_KEY` removed. + +## Mantle Messages (selector-gated) + +Record `migration_path: "mantle_messages"` only together with a `bedrock_models[].mantle_selector` +from `references/vendored/ai/bedrock-endpoint-policy.md` — normally `workspaces` (Anthropic-compatible +Workspaces exist only on `bedrock-mantle`). The client shape is the Step 2 repoint with +`base_url="https://bedrock-mantle.{region}.api.aws/anthropic"`, a Bedrock API key or token as the +string `api_key`, the bare `anthropic.claude-*` model ID in a Mantle-listed Region, and +`bedrock-mantle:*` IAM instead of `bedrock:InvokeModel*`. API continuity alone never selects Mantle; +the Runtime lane above already keeps the Anthropic SDK. ## Opus 5.5 target contract (verified 2026-09-24) diff --git a/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py b/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py index 1760e0cb6..0ab6ec546 100644 --- a/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py +++ b/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py @@ -23,6 +23,8 @@ PLUGIN_ROOT / "skills" / "azure-to-aws" / "references" / "vendored" / "ai" / "bedrock-endpoint-policy.md", ] +ANTHROPIC_GUIDE = PLUGIN_ROOT / "skills" / "shared" / "ai" / "ai-anthropic-to-bedrock.md" + DESIGN_AI_FILES = [ PLUGIN_ROOT / "skills" / "gcp-to-aws" / "references" / "phases" / "design" / "design-ai.md", PLUGIN_ROOT / "skills" / "azure-to-aws" / "references" / "phases" / "design" / "design-ai.md", @@ -96,6 +98,30 @@ def test_schema_docs_carry_the_runtime_default_vocabulary() -> None: assert value in text, f"{f}: mantle_selector enum value {value} missing" +def test_anthropic_guide_defaults_to_runtime_messages_with_a_string_token() -> None: + """The Anthropic guide keeps the SDK on bedrock-runtime /anthropic by default. + + https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html + (read 2026-11-07) passes the Bedrock token as a plain string `api_key`; the + pinned Anthropic SDK accepts no callable there, so the guide must never show + `api_key=lambda` / `auth_token=lambda`. Converse stays an explicit alternative. + """ + text = ANTHROPIC_GUIDE.read_text(encoding="utf-8") + assert "runtime_messages" in text, "Anthropic guide does not name runtime_messages" + assert '"converse"' in text, "Anthropic guide lost the explicit Converse alternative" + assert text.index("runtime_messages") < text.index('"converse"'), ( + "Anthropic guide must present runtime_messages before the converse alternative" + ) + assert "bedrock-runtime.{region}.amazonaws.com/anthropic" in text + assert 'required_client_api: "messages"' in text + assert "auth_token=lambda" not in text + assert "api_key=lambda" not in text + for f in DESIGN_AI_FILES: + assert "runtime_messages" in f.read_text(encoding="utf-8"), ( + f"{f}: Anthropic design line does not record migration_path runtime_messages" + ) + + def test_fixtures_record_a_selector_on_every_mantle_row() -> None: import json From b105531eef46a2823da823a2154e8cfc65f06478 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 7 Nov 2026 17:10:01 -0800 Subject: [PATCH 10/41] fix(aws-startup-advisor): fail closed on runtime_openai_cris plans without an API and add the Runtime Anthropic Messages lane to llm-to-bedrock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit normalize_api_path now raises for a runtime_openai_cris plan that recorded no source API instead of silently resolving to Converse; SKILL.md resolves that case with the user (resolved_runtime_api) and only an explicit required_client_api "converse" reaches the Converse rewrite. Adds the runtime_messages execution lane: create_messages_client / probe_runtime_messages_model (Anthropic SDK on bedrock-runtime /anthropic with a string token minted per client), --runtime-api messages, image_input.messages_message, the BedrockGetInferenceProfile IAM statement for CRIS targets, the rewriter's Runtime Anthropic Messages lane (Mantle express lane is now selector-gated, Converse rewrite is for explicit converse plans), evaluator §6b/§9.5b and the §10 messages branch, report-generator §6.4 and resolver notes, plus the endpoint policy section in SKILL.md. Pins anthropic>=0.75,<1 in the scripts project. Also un-indents the Converse snippet in the shared Anthropic guide so the agent-advisor snippet test executes the right block. Public sources (read 2026-11-07): https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-prereq.html https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html Verification (plugins/aws-startup-advisor): - uv run --project skills/llm-to-bedrock/scripts --with pytest python -m pytest skills/llm-to-bedrock/scripts -q -> 431 passed (418 + 13 new), 0 failed - uv run --project skills/agent-advisor/scripts --with pytest python -m pytest skills/agent-advisor/scripts -q -> 570 passed; git diff --stat 6b873f8c..HEAD -- skills/agent-advisor is empty - python3 tools/sync-vendored.py --check -> PASS 64 files - mise run ci -> exit 0 (lint:md 0 errors; lint:skill-refs 1517/0 missing; lint:vendored-parity 64; lint:artifacts 99/0; test 1408 passed + 222 subtests; test:telemetry 311; test:heroku-source-review 44) - root: python3 tools/validate.py -> All validations passed; mise run validate -> exit 0 - root: mise run build -> lint + validate pass; security fails only on the pre-existing gitleaks finding in upstream f8d09fd1 (tests/test_bedrock_migration_assessment_discover.py:118); not from this change - gitleaks protect --staged -> no leaks found - residual Mantle-default grep over plugins/*.md|*.json -> no hits --- .../agents/llm2bedrock-code-rewriter.md | 77 ++++++++-- .../agents/llm2bedrock-prompt-evaluator.md | 121 +++++++++++++-- .../agents/llm2bedrock-report-generator.md | 4 +- .../vendored/ai/ai-anthropic-to-bedrock.md | 21 +-- .../vendored/ai/ai-anthropic-to-bedrock.md | 21 +-- .../skills/llm-to-bedrock/SKILL.md | 61 ++++++-- .../resolve-bedrock-model-id.md | 4 + .../llm-to-bedrock/scripts/iam_policy.py | 17 ++- .../llm-to-bedrock/scripts/image_input.py | 32 +++- .../scripts/preflight_bedrock.py | 80 ++++++++-- .../llm-to-bedrock/scripts/pyproject.toml | 12 +- .../llm-to-bedrock/scripts/schemas/eval.json | 2 +- .../scripts/test_astra_execution_contract.py | 144 ++++++++++++++++-- .../llm-to-bedrock/scripts/test_iam_policy.py | 29 +++- .../scripts/test_image_input.py | 20 ++- .../scripts/test_preflight_bedrock.py | 48 ++++++ .../skills/llm-to-bedrock/scripts/uv.lock | 32 +++- .../shared/ai/ai-anthropic-to-bedrock.md | 21 +-- .../tests/test_endpoint_policy_contract.py | 13 +- 19 files changed, 657 insertions(+), 102 deletions(-) diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md b/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md index d46296670..bfbd08cff 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md @@ -147,8 +147,10 @@ Conservative scope — only universally-junk patterns; `.venv/` / `node_modules/ ## Strategy selection (check FIRST) If your context has `Rewrite strategy: runtime_openai`, use the runtime OpenAI lane below. -For `Rewrite strategy: mantle`, use the Mantle express lane. Otherwise use the Converse -rewrite. The selected `Target API surface` must match what the evaluator verified. +For `Rewrite strategy: runtime_messages`, use the Runtime Anthropic Messages lane. +For `Rewrite strategy: mantle`, use the Mantle express lane. Otherwise (an explicit `converse` +plan) use the Converse rewrite. The selected `Target API surface` must match what the +evaluator verified. ### Runtime OpenAI lane @@ -176,6 +178,59 @@ In §12 keep `openai>=2.45.0,<3` and `aws-bedrock-token-generator>=1,<2`; do not analyzer's generic OpenAI-to-boto3 dependency replacement to this lane. Apply confirmed deltas in §9, then skip the Converse-specific §8 rewrite guidance. +### Runtime Anthropic Messages lane + +Keep the Anthropic SDK and its Messages request/response shapes (`messages.create`, +`messages.stream`, `system=`, content blocks, tool use). Point the client at the runtime +endpoint `https://bedrock-runtime.{region}.amazonaws.com/anthropic` (the SDK appends +`/v1/messages`) with the exact validated profile ID from the plan, not a bare model ID +(except the documented eu-west-2 Opus 5.5 route) and not a Mantle URL. Public basis: + (read 2026-11-07). + +The pinned Anthropic SDK takes a **string** credential only — `api_key` is the Bedrock bearer +token minted by `provide_token`, **never a callable**, and the token has a bounded lifetime. +Mint the token per client, create the client at the request boundary, and close it when the +request is done; never keep one client alive past the token lifetime. For Python: + +```python +import anthropic +from aws_bedrock_token_generator import provide_token # aws-bedrock-token-generator + +REGION = "" + + +def bedrock_anthropic() -> anthropic.Anthropic: + """One client per request boundary; the token is a string minted right here.""" + return anthropic.Anthropic( + base_url=f"https://bedrock-runtime.{REGION}.amazonaws.com/anthropic", + api_key=provide_token(region=REGION), + ) + + +with bedrock_anthropic() as client: + r = client.messages.create(model="", max_tokens=1024, messages=msgs) + text = "".join(block.text for block in r.content if block.type == "text") + +# Streaming keeps the SDK's own helper: +with bedrock_anthropic() as client, client.messages.stream( + model="", max_tokens=1024, messages=msgs) as stream: + for chunk in stream.text_stream: + print(chunk, end="") +``` + +For a short CLI or one-shot script, `api_key=os.environ["AWS_BEARER_TOKEN_BEDROCK"]` is +acceptable (that token also expires within 12 hours); a server, worker, scheduled job, ECS +service or Lambda uses the factory above and `.env.example` does **not** set +`AWS_BEARER_TOKEN_BEDROCK`. Remove the old `ANTHROPIC_API_KEY` read. + +Runtime permissions are `bedrock:InvokeModel` and `bedrock:InvokeModelWithResponseStream` on +the inference-profile ARN and the foundation-model ARN(s) of its destination Regions, plus +`bedrock:GetInferenceProfile` on the profile ARN — not Mantle actions. In §12 keep `anthropic` +and add `aws-bedrock-token-generator>=1,<2`; do not apply an analyzer's generic +Anthropic-to-boto3 dependency replacement to this lane. Apply confirmed deltas in §9 (the +native `type: text` block shape and message history are preserved on this lane), then skip +the Converse-specific §8 rewrite guidance. + ### Mantle express lane The source SDK stays. Per client, change only three things: @@ -257,9 +312,9 @@ client = anthropic.Anthropic( Preserve compatible SDK request/response shapes. A model version change still requires §9 target checks, including typed response parsing for Opus 5.5. Apply the Chat Completions → Responses reshape only when that API change was selected. Preserve the exact validated runtime profile ID or ARN. After applying the changes above, skip the Converse-specific guidance in the rest of §8; the §9 behavior-delta application still applies normally. -**Converse for a proprietary GPT target is id-form-dependent, not family-dependent.** A **bare** proprietary GPT id (`openai.gpt-5.5`, `openai.gpt-5.4`, `openai.gpt-5.6-*`, `openai.gpt-6-*`) is a `bedrock-mantle` id with no `bedrock-runtime` in-region form — a boto3 `converse()` call against a bare id fails at runtime; if your context pairs a bare id with the Converse path, stop and report the contradiction. GPT-5.6, GPT-6 (Astra, Sol, Luna, 6.1 Sol) **and GPT-5.5 / GPT-5.4** (model cards read 2026-11-06: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`; Responses, Chat Completions and Converse on runtime, no Invoke) all have a `bedrock-runtime` path via CRIS ids (`us.`/`in.`/`global.` prefixed for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4 — those cards publish no `in.` profile): when the plan says `migration_path: runtime_openai_cris`, a Converse, Responses or Chat Completions rewrite against the CRIS id is valid (base URL `bedrock-runtime.{region}.amazonaws.com/openai/v1` for the OpenAI-compatible APIs; note Guardrails are Converse-only and prompt caching Responses-only on runtime). For a Responses-based source app, the mantle express lane above remains the default and smallest change. +**Converse for a proprietary GPT target is id-form-dependent, not family-dependent.** A **bare** proprietary GPT id (`openai.gpt-5.5`, `openai.gpt-5.4`, `openai.gpt-5.6-*`, `openai.gpt-6-*`) is a `bedrock-mantle` id with no `bedrock-runtime` in-region form — a boto3 `converse()` call against a bare id fails at runtime; if your context pairs a bare id with the Converse path, stop and report the contradiction. GPT-5.6, GPT-6 (Astra, Sol, Luna, 6.1 Sol) **and GPT-5.5 / GPT-5.4** (model cards read 2026-11-06: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`; Responses, Chat Completions and Converse on runtime, no Invoke) all have a `bedrock-runtime` path via CRIS ids (`us.`/`in.`/`global.` prefixed for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4 — those cards publish no `in.` profile): when the plan says `migration_path: runtime_openai_cris`, a Converse, Responses or Chat Completions rewrite against the CRIS id is valid (base URL `bedrock-runtime.{region}.amazonaws.com/openai/v1` for the OpenAI-compatible APIs; note Guardrails are Converse-only and prompt caching Responses-only on runtime). The Runtime OpenAI lane is the default for a compatible source; use the Mantle express lane only when the plan recorded a `mantle_*` path with a `mantle_selector`. -### Converse rewrite (default) +### Converse rewrite (explicit `converse` plans) Choose the rewrite approach based on framework: @@ -887,11 +942,15 @@ grep -rl "from openai\|import openai\|require.*openai\|from anthropic\|import an If any files still contain source SDK references, fix them before proceeding. Test directories are NOT excluded from this scan on purpose: the source SDK package is being removed from the manifest, so a leftover `import openai` in a customer test means `pytest` ImportErrors on the customer's machine — §18.0 should have migrated those tests; if one appears here, go back and fix it. -**OpenAI-compatible lane exception:** when this run used `Rewrite strategy: mantle` or -`Rewrite strategy: runtime_openai`, the source-SDK imports are expected to remain, so this -residual scan does not apply. Verify the selected endpoint's `base_url`, Bedrock credentials -and model identifiers instead: bare IDs for Mantle, exact validated profiles for runtime. -The runtime lane must retain the Chat/Responses API verified by the evaluator. +**Source-SDK lane exception:** when this run used `Rewrite strategy: mantle`, +`Rewrite strategy: runtime_openai` or `Rewrite strategy: runtime_messages`, the source-SDK +imports are expected to remain, so this residual scan does not apply. Verify the selected +endpoint's `base_url`, Bedrock credentials and model identifiers instead: bare IDs for Mantle, +exact validated profiles for runtime. The runtime OpenAI lane must retain the Chat/Responses +API verified by the evaluator. For `runtime_messages`, Anthropic imports are expected: verify +the `base_url` ends at `bedrock-runtime..amazonaws.com/anthropic`, the credential is a +string Bedrock token (no callable, no `ANTHROPIC_API_KEY`), and the model is the exact +validated profile ID. - **Short CLI / one-shot script:** `base_url` set to the Mantle path and `api_key=os.environ["AWS_BEARER_TOKEN_BEDROCK"]`. - **Server, worker, scheduled job, ECS service, or Lambda:** `BedrockOpenAI(..., bedrock_token_provider=lambda: provide_token(...))` (or the equivalent auto-refreshing client for the SDK in use), **no** `AWS_BEARER_TOKEN_BEDROCK` read in code, and `.env.example` does **not** set it. diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-prompt-evaluator.md b/plugins/aws-startup-advisor/agents/llm2bedrock-prompt-evaluator.md index d933c4b3e..d89eb6169 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-prompt-evaluator.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-prompt-evaluator.md @@ -41,7 +41,7 @@ Read from prompt context (forwarded from llm2bedrock-code-analyzer, llm2bedrock- - **``** — `/.saws-migrate/golden-dataset/templates/prompt_template.txt` (from T2-2). - **``** — the exact validated target ID or ARN from the migration plan. It identifies the endpoint, not the API: bare proprietary GPT uses Mantle; supported CRIS uses runtime. `` selects Chat, Responses or Converse within that endpoint. Preserve both through every call; never remove a profile prefix to change the endpoint. - **``** — AWS region for Bedrock (the `AWS region:` line in your context). -- **``** — copy the normalized `Target API surface` context line: `chat_completions`, `responses`, or `converse`. It applies to both Mantle and runtime. If a direct legacy dispatch omits that line, call `preflight_bedrock.normalize_api_path` with the saved path, target IDs and `runtime_api`, then derive the same surface. An absent runtime API means Converse; never discard an explicit runtime Chat/Responses choice. +- **``** — copy the normalized `Target API surface` context line: `chat_completions`, `responses`, `messages` (the runtime Anthropic Messages API, for a `runtime_messages` plan), or `converse`. It applies to both Mantle and runtime. If a direct legacy dispatch omits that line, call `preflight_bedrock.normalize_api_path` with the saved path, target IDs and `runtime_api`, then derive the same surface. A `runtime_openai_cris` plan with no recorded API makes that call raise — stop and have the orchestrator resolve the source API; never discard an explicit runtime Chat/Responses/Messages choice and never substitute Converse. - **From `llm2bedrock-code-analyzer` (`AiAnalysisData`)** — key fields: - `source_provider` — `openai` / `anthropic` / `google` / `cohere` / `custom`. Drives §9 baseline gating. (Vertex AI customers are emitted as `google` here; the analyzer's `errors` field carries the `vertex AI auth detected` signal that gates baseline collection upstream — by the time you reach §9, `source_baseline_available` already reflects that.) - `source_models` — list of source-model IDs. Pass `` to the §9 baseline skill verbatim. @@ -135,7 +135,8 @@ call in §6, §9.5 and §10:** | --------------------------------------------------------- | ------------------- | ------------ | ------------ | ------------------------------- | | bare `openai.gpt-5*` (not `-oss`) or `openai.gpt-6*` | Selected Mantle API | §6a | §9.5a | §8 if same-model; otherwise §10 | | Supported CRIS with selected Chat/Responses | Runtime OpenAI API | §6a | §9.5a | §8 if same-model; otherwise §10 | -| Supported CRIS with Converse (legacy default) | `boto3.converse` | §6 | §9.5 | §8 if same-model; otherwise §10 | +| Supported profile with selected `messages` | Runtime Anthropic Messages | §6b | §9.5b | §8 if same-model; otherwise §10 | +| Supported CRIS with explicit Converse | `boto3.converse` | §6 | §9.5 | §8 if same-model; otherwise §10 | | anything else (Claude, Nova, `openai.gpt-oss-*`) | `boto3.converse` | §6 | §9.5 | §10, or §8 if same-model | The bare proprietary GPT ids select the `bedrock-mantle` endpoint; supported CRIS ids select runtime. Calling `boto3.converse` against a bare Mantle id fails, so running the Converse @@ -144,8 +145,9 @@ check below for a mantle target would abort this agent before it ever reached This choice is made once and holds for every Bedrock call in the run. It is independent of `same_model_family`: the short-circuit decides whether to _score_, -while this table decides _how to call_. A same-model Anthropic run still uses -Converse, and a same-model OpenAI run still needs its vision smoke test. +while this table decides _how to call_. A same-model Anthropic run still uses the +selected surface (Messages on a `runtime_messages` plan, Converse on an explicit `converse` +plan), and a same-model OpenAI run still needs its vision smoke test. ```bash mkdir -p /.saws-migrate/eval-results @@ -250,6 +252,52 @@ Because these targets are same-model migrations, §8's short-circuit applies rig after §7: run each golden prompt through this same Responses client for connectivity verification rather than the Converse script in §10. +# 6b. Runtime Anthropic Messages connectivity + +Use this INSTEAD of §6 when `` is `messages` (a `runtime_messages` plan: +the Anthropic SDK stays and talks to `https://bedrock-runtime..amazonaws.com/anthropic` +with a string Bedrock token — public basis +, read 2026-11-07). +The client factory lives in `preflight_bedrock.create_messages_client`; it mints the token per +client, so create the client inside a `with` block and let it close. + +```bash +mkdir -p /.saws-migrate/eval-results + +AWS_REGION= when your context has an `AWS profile` line> uv run --project python - <<'PY' +import os, sys +sys.path.insert(0, "") +from preflight_bedrock import create_messages_client +region = os.environ.get('AWS_REGION', 'us-east-1') +try: + with create_messages_client(region, max_retries=2) as c: + r = c.messages.create(model='', max_tokens=16, + messages=[{'role': 'user', 'content': 'ping'}]) + text = ''.join(block.text for block in r.content if block.type == 'text') + print('OK:', text or '(invocation succeeded; no text)', f'stop_reason={r.stop_reason}') +except Exception as e: + print(f'FAIL [{type(e).__name__}]: {e}', file=sys.stderr) + sys.exit(1) +PY +``` + +Interpret the result: + +- **Exit 0 + "OK:" line** — proceed to §7. +- **HTTP 404 / not-found** — the selected profile is not available in `` on the + runtime Messages endpoint. Load `resolve-bedrock-model-id` with the plan's ID and the region + (it resolves `runtime_messages` like Converse: a supported `us.`/`eu.`/`global.` profile), retry + with the returned ID, and otherwise return `{ blocked: { reason: 'model_unresolvable', detail: ' — is not available on the bedrock-runtime Anthropic Messages endpoint in ' } }`. +- **HTTP 401 / 403** — distinguish the two causes from the message. If it names model access, + return `{ blocked: { reason: 'model_access', ... } }` as in §6. Otherwise it is IAM on the + runtime endpoint: the principal needs `bedrock:InvokeModel` / `bedrock:InvokeModelWithResponseStream` + on the inference-profile and foundation-model ARNs plus `bedrock:GetInferenceProfile` on the + profile — **not** Mantle actions. Return `{ blocked: { reason: 'authz', detail: ' — grant bedrock:InvokeModel* on the profile and foundation-model ARNs and bedrock:GetInferenceProfile on the profile.' } }`. +- **`ModuleNotFoundError`** — the pinned scripts environment is missing `anthropic` or + `aws-bedrock-token-generator` (both declared in `scripts/pyproject.toml`). Return + `{ blocked: { reason: 'messages_deps_missing', detail: ' — re-sync the pinned environment.' } }`. Do NOT treat this as a passing check and do NOT substitute Converse. +- **Any other failure** — surface the exact error type and message and STOP, as in §6. + # 7. Load golden dataset ```bash @@ -482,11 +530,48 @@ Outcomes are the same as §9.5: `VISION_OK` → proceed; `VISION_INFRA_SKIPPED` **On `VISION_FAIL` specifically, do not fall back to text-only prompts.** The migration's vision capability is unverified at that point, and a text-only pass rate would misrepresent it as working. The content-block shapes above match their selected OpenAI API; if it is rejected for a reason other than access, report the exact error rather than guessing at an alternative shape. +# 9.5b Runtime Anthropic Messages vision smoke test + +Use this INSTEAD of §9.5 when `` is `messages` AND `special_patterns.vision == true`. The image goes through `image_input.messages_message` as a base64 `source` block (the Messages shape), never as an OpenAI data URL. + +```bash +AWS_REGION= when your context has an `AWS profile` line> uv run --project python - <<'PY' +import os, sys +try: + import urllib.request + img = urllib.request.urlopen( + "https://upload.wikimedia.org/wikipedia/commons/thumb/4/4d/Cat_November_2010-1a.jpg/320px-Cat_November_2010-1a.jpg", + timeout=15, + ).read() +except Exception as e: + print(f"VISION_INFRA_SKIPPED [{type(e).__name__}]: {e}", file=sys.stderr) + sys.exit(0) + +sys.path.insert(0, "") +from preflight_bedrock import create_messages_client +from image_input import messages_message +region = os.environ.get("AWS_REGION", "us-east-1") +try: + with create_messages_client(region, max_retries=2) as client: + r = client.messages.create( + model="", max_tokens=32, + messages=[messages_message("Describe this image briefly.", "smoke.jpg", img)]) + text = "".join(block.text for block in r.content if block.type == "text") + print("VISION_OK:", text) +except Exception as e: + print(f"VISION_FAIL [{type(e).__name__}]: {e}", file=sys.stderr) + sys.exit(1) +PY +``` + +Outcomes are the same as §9.5: `VISION_OK` → proceed; `VISION_INFRA_SKIPPED` → record `vision_smoke_skipped` in `notes` and proceed; `VISION_FAIL` → surface the exact error and STOP, routing a model-access denial through `{ blocked: { reason: 'model_access', ... } }` and an IAM denial through `{ blocked: { reason: 'authz', ... } }` per §6b. Do not fall back to text-only prompts. + # 10. Run golden prompt evaluation -> **Use the endpoint and API selected in §6.** A model change (including Pro-to-Astra) -> still requires golden evaluation and scoring on Mantle. Same-model runs use §8's -> connectivity-only branch. Never send a bare Astra id to Converse. +> **Use the endpoint and API selected in §6 / §6a / §6b.** A model change (including +> Pro-to-Astra) still requires golden evaluation and scoring on the selected endpoint. +> Same-model runs use §8's connectivity-only branch. Never send a bare Astra id to Converse, +> and never send a `messages` run through Converse. For each prompt in the golden dataset, run the evaluation via `python` stdin (avoids the brittle nested-heredoc + escaped-quote pattern that breaks on any literal `'` inside the script): @@ -503,19 +588,22 @@ import time import boto3 from botocore.exceptions import ClientError sys.path.insert(0, "") -from preflight_bedrock import is_mantle_model, create_openai_client -from image_input import converse_message, responses_message, chat_message +from preflight_bedrock import is_mantle_model, create_openai_client, create_messages_client +from image_input import converse_message, responses_message, chat_message, messages_message region = os.environ.get("AWS_REGION", "us-east-1") model_id = "" api_surface = "" -if api_surface not in {"responses", "chat_completions", "converse"}: +if api_surface not in {"responses", "chat_completions", "converse", "messages"}: raise ValueError("Resolve the selected API surface before evaluation") openai_api = api_surface in {"responses", "chat_completions"} +messages_api = api_surface == "messages" if is_mantle_model(model_id) and not openai_api: raise ValueError("A bare Mantle model cannot use Converse") if openai_api: client = create_openai_client(model_id, region, max_retries=0) +elif messages_api: + pass # a Messages client is created per request inside target_with_backoff (string token) else: bedrock = boto3.client("bedrock-runtime", region_name=region) @@ -569,6 +657,19 @@ def target_with_backoff(prompt): if prompt.get("system_prompt"): args["instructions"] = prompt["system_prompt"] return client.responses.create(**args).output_text + if messages_api: + with create_messages_client(region) as messages_client: + r = messages_client.messages.create( + model=model_id, max_tokens=4096, + messages=[messages_message(prompt["user_prompt"], prompt.get("image_path"), raw)], + **({"system": prompt["system_prompt"]} if prompt.get("system_prompt") else {})) + text = "".join(block.text for block in r.content if block.type == "text") + stop_reason = getattr(r, "stop_reason", "unknown") + if stop_reason in ("max_tokens", "refusal", "tool_use"): + raise ValueError(f"non_final_response (stop_reason={stop_reason})") + if not text: + raise ValueError(f"no_text_response (stop_reason={stop_reason})") + return text system = [{"text": prompt["system_prompt"]}] if prompt.get("system_prompt") else [] r = bedrock.converse(modelId=model_id, messages=[converse_message(prompt["user_prompt"], prompt.get("image_path"), raw)], diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md index 9dd8b8107..8ca2d0bdb 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md @@ -571,7 +571,9 @@ uv run --project python /iam_policy.py \ --output /.saws-migrate/iam-policy.json ``` -The script handles the dual-ARN pattern: foundation-model ARNs for plain model IDs and inference-profile ARNs for geo-prefixed IDs (e.g. `us.anthropic.claude-sonnet-4-6`). For a GPT-6 CRIS id and for the GPT-5.5 / GPT-5.4 CRIS ids (`us.`/`global.openai.gpt-5.5`, `us.`/`global.openai.gpt-5.4`, or their `inference-profile/` ARNs) it adds a third resource, `arn:aws:bedrock:::project/default`, because those model cards (GPT-5.4 / GPT-5.5 read 2026-11-07) require `bedrock:InvokeModel` on the default project in addition to the inference profile; do not strip it from the report's policy excerpt. The output is a ready-to-use IAM policy JSON file. +The script handles the dual-ARN pattern: foundation-model ARNs for plain model IDs and inference-profile ARNs for geo-prefixed IDs (e.g. `us.anthropic.claude-sonnet-4-6`). For any cross-region (CRIS) target it also emits a `BedrockGetInferenceProfile` statement (`bedrock:GetInferenceProfile` on the profile ARNs) — the documented prerequisite for invoking through a profile (, read 2026-11-07). For a GPT-6 CRIS id and for the GPT-5.5 / GPT-5.4 CRIS ids (`us.`/`global.openai.gpt-5.5`, `us.`/`global.openai.gpt-5.4`, or their `inference-profile/` ARNs) it adds a third resource, `arn:aws:bedrock:::project/default`, because those model cards (GPT-5.4 / GPT-5.5 read 2026-11-07) require `bedrock:InvokeModel` on the default project in addition to the inference profile; do not strip it from the report's policy excerpt. The output is a ready-to-use IAM policy JSON file. + +**Runtime OpenAI and Runtime Messages lanes need no Mantle statement.** A `Rewrite strategy: runtime_openai` or `Rewrite strategy: runtime_messages` run keeps the source SDK but authorizes through `bedrock:InvokeModel*` (+ `GetInferenceProfile`, and the default-project ARN for the GPT-6 / GPT-5.5 / GPT-5.4 CRIS ids above) on the `bedrock-runtime` endpoint; report exactly the statements the script produced and do not add `bedrock-mantle:*` for them. Per-token model pricing is the same on both endpoints (, read 2026-11-07), so the endpoint choice does not change the §6.3 cost table. **Mantle targets get different actions.** When any target is a bare proprietary OpenAI GPT model (GPT-5.x or `openai.gpt-6-astra`), the script also emits `bedrock-mantle:CreateInference` / `Get*` / `List*` scoped to `arn:aws:bedrock-mantle:::project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` (AWS does not permit narrowing that one). `bedrock:InvokeModel` alone does **not** authorize these models. Two consequences to carry into the report: diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md index 59d6a0796..05cbdf73d 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md @@ -89,8 +89,8 @@ Response parsing keeps the native shape: select text blocks by `type == "text"`, position, because thinking and tool-use blocks can appear first. ```python -text = "".join(block.text for block in response.content if block.type == "text") -if not text or response.stop_reason in ("max_tokens", "refusal", "tool_use"): +answer = "".join(block.text for block in response.content if block.type == "text") +if not answer or response.stop_reason in ("max_tokens", "refusal", "tool_use"): raise ValueError(f"No complete text response (stop_reason={response.stop_reason})") ``` @@ -154,15 +154,16 @@ metadata). This is a rewrite, not a repoint: Content becomes typed blocks `[{"text": "..."}]`; `max_tokens` moves to `inferenceConfig.maxTokens`; select text blocks by their `text` key because reasoning and tool blocks can appear first: - ```python - text = "".join(block["text"] for block in response["output"]["message"]["content"] if "text" in block) - stop_reason = response.get("stopReason", "unknown") - if not text or stop_reason in ("max_tokens", "model_context_window_exceeded", "guardrail_intervened", "content_filtered", "refusal", "tool_use"): - raise ValueError(f"No complete text response (stopReason={stop_reason})") - ``` +```python +text = "".join(block["text"] for block in response["output"]["message"]["content"] if "text" in block) +stop_reason = response.get("stopReason", "unknown") +if not text or stop_reason in ("max_tokens", "model_context_window_exceeded", "guardrail_intervened", "content_filtered", "refusal", "tool_use"): + raise ValueError(f"No complete text response (stopReason={stop_reason})") +``` + +For tool loops on Converse, retain the original `response["output"]["message"]` in assistant +history, including signed reasoning blocks, and handle every tool request before continuing. - For tool loops, retain the original `response["output"]["message"]` in assistant history, - including signed reasoning blocks, and handle every tool request before continuing. - System prompts: `system="You are helpful"` → `system=[{"text": "You are helpful"}]`. - Streaming: `client.messages.stream()` → `client.converse_stream()`; iterate `response["stream"]` for `contentBlockDelta` events. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md index 59d6a0796..05cbdf73d 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-anthropic-to-bedrock.md @@ -89,8 +89,8 @@ Response parsing keeps the native shape: select text blocks by `type == "text"`, position, because thinking and tool-use blocks can appear first. ```python -text = "".join(block.text for block in response.content if block.type == "text") -if not text or response.stop_reason in ("max_tokens", "refusal", "tool_use"): +answer = "".join(block.text for block in response.content if block.type == "text") +if not answer or response.stop_reason in ("max_tokens", "refusal", "tool_use"): raise ValueError(f"No complete text response (stop_reason={response.stop_reason})") ``` @@ -154,15 +154,16 @@ metadata). This is a rewrite, not a repoint: Content becomes typed blocks `[{"text": "..."}]`; `max_tokens` moves to `inferenceConfig.maxTokens`; select text blocks by their `text` key because reasoning and tool blocks can appear first: - ```python - text = "".join(block["text"] for block in response["output"]["message"]["content"] if "text" in block) - stop_reason = response.get("stopReason", "unknown") - if not text or stop_reason in ("max_tokens", "model_context_window_exceeded", "guardrail_intervened", "content_filtered", "refusal", "tool_use"): - raise ValueError(f"No complete text response (stopReason={stop_reason})") - ``` +```python +text = "".join(block["text"] for block in response["output"]["message"]["content"] if "text" in block) +stop_reason = response.get("stopReason", "unknown") +if not text or stop_reason in ("max_tokens", "model_context_window_exceeded", "guardrail_intervened", "content_filtered", "refusal", "tool_use"): + raise ValueError(f"No complete text response (stopReason={stop_reason})") +``` + +For tool loops on Converse, retain the original `response["output"]["message"]` in assistant +history, including signed reasoning blocks, and handle every tool request before continuing. - For tool loops, retain the original `response["output"]["message"]` in assistant history, - including signed reasoning blocks, and handle every tool request before continuing. - System prompts: `system="You are helpful"` → `system=[{"text": "You are helpful"}]`. - Streaming: `client.messages.stream()` → `client.converse_stream()`; iterate `response["stream"]` for `contentBlockDelta` events. diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md b/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md index 18f188d0b..e2897cd7b 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md @@ -497,6 +497,20 @@ extrapolated cost table (`llm2bedrock-report-generator.md` §6.3) run exactly as --- +## Endpoint policy (bedrock-runtime default) + +`bedrock-runtime` is the default endpoint for new applications and for every migration +lane in this skill (Converse, the runtime OpenAI-compatible APIs and the runtime Anthropic +Messages API). `bedrock-mantle` is selected only when the saved plan records a verified +`mantle_selector`: background inference, server-side/preconfigured tools (including Web +Search), customer-created Projects or Workspaces, a model or API that exists only on Mantle, +or strict in-Region processing where Runtime offers only a cross-region profile. API +continuity alone never selects Mantle — keeping the OpenAI or Anthropic SDK is a Runtime +lane. See `skills/shared/ai/bedrock-endpoint-policy.md` (vendored at +`$ASSESS_BASE/references/vendored/ai/bedrock-endpoint-policy.md`) for the public sources. + +--- + ## Phase A — Assess (runs $ASSESS_SKILL's own AI-path files, inline) **Do NOT read source code, detect AI SDKs, or ask Clarify questions yourself from scratch.** @@ -817,11 +831,19 @@ reliably visible, and even that must be materialized to the file for the baselin ### B4 — Bedrock preflight ```bash -uv run --project $SCRIPTS python $SCRIPTS/preflight_bedrock.py --region $REGION --models --dataset-size 200 --runtime-api +uv run --project $SCRIPTS python $SCRIPTS/preflight_bedrock.py --region $REGION --models --dataset-size 200 --runtime-api ``` (`--dataset-size 200` matches the golden-dataset cap, so the quota warning reflects the worst case. Prefix with `AWS_PROFILE=$AWS_PROFILE_CHOICE` if B2 chose a non-default profile.) +`--runtime-api` is the selected row's `required_client_api`: pass `messages` when it is +`"messages"` (a `runtime_messages` plan — the probe uses the Anthropic SDK against the runtime +`/anthropic` endpoint), `responses` or `chat_completions` for the runtime OpenAI-compatible +APIs, and `converse` only when the plan recorded `"converse"`. For a `runtime_openai_cris` plan +ALWAYS pass the explicit source API (`responses` | `chat_completions`, or `converse` only when +recorded) — never omit the flag to fall back to the Converse default; a plan with no recorded +API is resolved first by the context-block rule below, not guessed here. + Parse the JSON output. On failure the TOP LEVEL carries `reason`/`detail` (lifted from the first failing model) plus `failing_models` (all failing ids); per-model verdicts are in `models[]`: @@ -830,6 +852,7 @@ first failing model) plus `failing_models` (all failing ids); per-model verdicts - `ok == false` + `reason: authz` → IAM denies inference. For a Converse/InvokeModel target the action to grant is `bedrock:InvokeModel`; for a bare proprietary GPT target (GPT-5.x or `openai.gpt-6-astra`) it is the `bedrock-mantle:*` set (see B4a). The `detail` names which. Tell the user the action to grant; stop. - `ok == false` + `reason: mantle_deps_missing` → the pinned scripts environment lacks `openai` / `aws-bedrock-token-generator`, so a mantle-only target could not be probed at all. This is an environment fault, not a Bedrock verdict: tell the user to re-sync (`uv sync --project $SCRIPTS`) and stop. Do NOT proceed — access was never verified. - `ok == false` + `reason: openai_deps_missing` → the selected runtime Chat/Responses API could not be probed because its SDK is missing. Re-sync the same scripts environment and stop; do not substitute Converse. +- `ok == false` + `reason: messages_deps_missing` → the selected runtime Anthropic Messages API could not be probed because `anthropic` / `aws-bedrock-token-generator` is missing from the pinned scripts environment. Re-sync (`uv sync --project $SCRIPTS`) and stop; do not substitute Converse — access was never verified. - `ok == false` + `reason: model_unavailable` → Read the `resolve-bedrock-model-id` reference at `$HELPERS/resolve-bedrock-model-id/resolve-bedrock-model-id.md` and follow its procedure with each ID from `failing_models` + region. AskUserQuestion with the candidates: "Use `` (cross-region inference profile)" / "Paste a different model ID" / "Abort". On a choice, replace the ID in `$TARGET_MODELS` and re-run B4. - `ok == false` + any other `reason` → show `detail` and stop. - `ok == true` → proceed. Surface any `quota_warning`, and any model whose `reason` is @@ -881,11 +904,18 @@ resolved_api_path = normalize_api_path( Read `plan_runtime_api` from the selected model row's `required_client_api`. For older plans, fall back to that row's `runtime_api`, then `ai_architecture.code_migration.runtime_api`. -A current Design CRIS plan without an explicit API uses `responses`; only a legacy -Converse plan retains `None`. An explicit `invoke` surface is unsupported by this -execution lane: stop for an explicit supported API decision rather than treating it as Converse. A `runtime_openai_cris` -plan resolves to `runtime_openai_chat` or `runtime_openai_responses` when that API is explicit, -and retains the Converse default otherwise. Do not ignore this field on dispatch or resume. +An explicit `invoke` surface is unsupported by this execution lane: stop for an explicit +supported API decision rather than treating it as Converse. A `runtime_openai_cris` plan +resolves to `runtime_openai_chat` or `runtime_openai_responses` from that explicit API and +**fails closed when none is recorded**: `normalize_api_path` raises `ValueError` ("Converse is +never the implicit default"). When that happens, do not catch it and guess — AskUserQuestion +"Which API does the source app call: Responses or Chat Completions?", record the answer in +run-context as `resolved_runtime_api`, use it as `plan_runtime_api` and re-normalize (on +resume, read `resolved_runtime_api` from run-context before asking again). An explicit +`required_client_api: "converse"` is the ONLY route to a Converse rewrite for a +`runtime_openai_cris` plan. `runtime_messages` (Anthropic SDK on the runtime `/anthropic` +endpoint) and `mantle_messages` pass through `normalize_api_path` unchanged. Do not ignore +this field on dispatch or resume. The legacy `migration_path: "mantle"` alias with bare proprietary GPT targets resolves to `mantle_openai_responses`, so evaluator and rewriter keep the documented Responses default. Other explicit `migration_path` values are preserved. With no path, an all-bare proprietary GPT target @@ -903,7 +933,7 @@ AWS region: <$REGION> AWS profile (pass as --profile / AWS_PROFILE= inline on every aws/boto3 invocation): <$AWS_PROFILE_CHOICE — omit line if default> Target Bedrock model(s): Target API path: -Target API surface: +Target API surface: Migration plan dir: <$MIGRATION_DIR> Resolved target model id: Scripts directory (pinned uv toolchain): <$SCRIPTS> @@ -1093,9 +1123,12 @@ Below, AskUserQuestion: **Gate (a.5) — Rewrite strategy (from the normalized API path).** Reuse `resolved_api_path` from the context-block normalization above. If it starts with `mantle`, set `rewrite_strategy = "mantle"`. For `runtime_openai_chat` or `runtime_openai_responses`, -set `rewrite_strategy = "runtime_openai"`; otherwise set `rewrite_strategy = "converse"`. +set `rewrite_strategy = "runtime_openai"`. For `runtime_messages`, +set `rewrite_strategy = "runtime_messages"`; otherwise set `rewrite_strategy = "converse"`. An absent legacy plan field is not a second Converse default here: a validated bare Astra -plan without that field has already resolved to Mantle Responses before C3. Explicit +plan without that field has already resolved to Mantle Responses before C3, and a +`runtime_openai_cris` plan without an API has already failed closed (and been resolved by +the user) in the context-block normalization. Explicit `mantle_openai_chat`, `mantle_openai_responses`, and runtime CRIS choices stay unchanged. **Gate (b) — Behavior-delta resolution.** For each `analysis.behavior_deltas[]` with @@ -1141,6 +1174,12 @@ When `rewrite_strategy == "runtime_openai"`, add `Rewrite strategy: runtime_open C5 must use the shared `Target API surface` and exact target profile at the runtime `/openai/v1` endpoint; it must not use the Mantle client or a Converse rewrite. +When `rewrite_strategy == "runtime_messages"`, add `Rewrite strategy: runtime_messages`. +C5 keeps the Anthropic SDK and its Messages request/response shapes on the runtime +`/anthropic` endpoint (`https://bedrock-runtime.{region}.amazonaws.com/anthropic`) with the +exact validated profile ID and a STRING Bedrock token (`provide_token(region=...)` minted per +client, never a callable); it must not use the Mantle URL or a Converse rewrite. + Rebuild these context lines from the same normalized path on every dispatch/resume. The existing `assess_design_sha256` run-context check detects a changed model/API decision. If a bare Astra id is paired with a runtime path, or a CRIS id with Mantle, stop and correct the plan; @@ -1197,6 +1236,10 @@ Tell the user, before collecting models: ### AC2 — Collect requested models and region (do NOT resolve friendly names yet) +- **Default endpoint for new work is `bedrock-runtime`:** when the user has not yet chosen an + ID form, offer the runtime form (bare foundation-model ID or a supported `us.`/`global.` + cross-region profile) first; a bare proprietary GPT ID (`bedrock-mantle`) is checked only + when the user names it or has a verified Mantle-only need (see the Endpoint policy section). - **Models (raw request only):** if `$ARGUMENTS` names model IDs, use them directly — skip resolution, they're already IDs. Otherwise **AskUserQuestion**: "Which models do you want access to? Give Bedrock model IDs (e.g. `anthropic.claude-sonnet-4-5-v1:0`, diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/resolve-bedrock-model-id/resolve-bedrock-model-id.md b/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/resolve-bedrock-model-id/resolve-bedrock-model-id.md index f2da1fd11..3ed65306a 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/resolve-bedrock-model-id/resolve-bedrock-model-id.md +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/references/helpers/resolve-bedrock-model-id/resolve-bedrock-model-id.md @@ -229,6 +229,10 @@ stops on abort. GPT-5.5/5.4 CRIS ids flow through Steps 1–5 like any other inference profile. See `$ASSESS_BASE/references/shared/openai-on-bedrock.md` for the authoritative family split. +- A `runtime_messages` plan (Anthropic SDK on the `bedrock-runtime` `/anthropic` endpoint) + resolves exactly like Converse: a supported `us.`/`eu.`/`global.` inference profile through + Steps 1–5, with a bare id only for the documented eu-west-2 Opus 5.5 route (Step 0). + `mantle_messages` uses the bare `anthropic.claude-*` id (Step 0, mantle endpoint). - Output of this skill should replace the plan's `target_model_id` in the caller's context — downstream phases (evaluator, rewriter) receive the validated ID only. diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py index ce486b3ae..9e194569e 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py @@ -110,9 +110,14 @@ def requires_default_project_grant(model_id: str) -> bool: def generate_policy(model_ids: list[str], region: str, account_id: str) -> dict: """Build a scoped IAM policy covering exactly the given model IDs. - Emits up to three statements, depending on which endpoints the targets use: + Emits up to four statements, depending on which endpoints the targets use: - `bedrock:InvokeModel*` scoped to foundation-model ARNs (plain IDs) and inference-profile ARNs (geo-prefixed IDs), for bedrock-runtime targets + - `bedrock:GetInferenceProfile` scoped to the inference-profile ARNs, when any + runtime target is a cross-region inference profile — the documented + prerequisite for invoking through a profile + (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-prereq.html, + read 2026-11-07) - `bedrock-mantle:CreateInference` / `Get*` / `List*` scoped to the account's mantle projects, for mantle-only targets (SigV4 auth) - `bedrock-mantle:CallWithBearerToken` on `*`, for Bedrock API-key auth @@ -156,6 +161,16 @@ def generate_policy(model_ids: list[str], region: str, account_id: str) -> dict: "Resource": sorted(set(resources)), }) + profile_arns = [inference_profile_arn(mid, region, account_id) + for mid in runtime_ids if is_inference_profile(mid)] + if profile_arns: + statements.append({ + "Sid": "BedrockGetInferenceProfile", + "Effect": "Allow", + "Action": ["bedrock:GetInferenceProfile"], + "Resource": sorted(set(profile_arns)), + }) + if mantle_ids: statements.append({ "Sid": "BedrockMantleInference", diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/image_input.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/image_input.py index e8ec6567e..115afcbc0 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/image_input.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/image_input.py @@ -1,4 +1,5 @@ -"""Build image-bearing request payloads for Converse, Responses and Chat Completions. +"""Build image-bearing request payloads for Converse, Responses, Chat Completions and +the Anthropic Messages API. Two mistakes are easy to make by hand and both produce a confusing failure mode where the vision smoke test passes (it hardcodes a known-good jpeg) while every @@ -13,8 +14,13 @@ declares `ResponseInputImageParam.detail` as `Required`, so omitting it is a contract violation even though a TypedDict will not catch it at runtime. -Callers should use `converse_message` / `responses_message` / `chat_message` rather than assembling -these dicts inline, so the shape is decided in one tested place. +4. **Messages images are a base64 `source` block, not a data URL.** The Anthropic + Messages API takes `{"type": "image", "source": {"type": "base64", "media_type": + "image/jpeg", "data": ""}}`; the OpenAI-style `image_url` shape is rejected. + +Callers should use `converse_message` / `responses_message` / `chat_message` / +`messages_message` rather than assembling these dicts inline, so the shape is decided +in one tested place. """ from pathlib import Path @@ -100,3 +106,23 @@ def chat_message(prompt: str, image_path: str | None = None, content.append({"type": "image_url", "image_url": { "url": data_url(image_path, raw)}}) return {"role": "user", "content": content} + + +def messages_message(prompt: str, image_path: str | None = None, + raw: bytes | None = None) -> dict: + """One Anthropic Messages `messages[]` item. Image block precedes the text block. + + The image is a base64 `source` block with the normalized MIME type — the same + `.jpg` -> `image/jpeg` rule as the other builders; the Messages API rejects + `image/jpg`. + """ + import base64 + content: list[dict] = [] + if image_path is not None: + if raw is None: + raise ValueError("raw image bytes are required when image_path is given") + content.append({"type": "image", "source": { + "type": "base64", "media_type": mime_type(image_path), + "data": base64.b64encode(raw).decode()}}) + content.append({"type": "text", "text": prompt}) + return {"role": "user", "content": content} diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py index fde343ece..f2fd9d059 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py @@ -6,12 +6,14 @@ Always prints JSON — including for missing credentials / bad region — so the caller can parse the verdict instead of a traceback. On failure the top level carries `reason`/`detail`/`failing_models` lifted from the first failing model. -Chat models default to Converse; --runtime-api preserves an explicit proprietary -GPT Chat/Responses selection. Embedding models (which don't speak -Converse) via InvokeModel with their family's request body. Bare OpenAI -proprietary GPT ids are mantle-served and probed via the OpenAI Responses API; -GPT-6 Astra (us./global.) and GPT-5.6 (us./in./global.) CRIS ids are bedrock-runtime targets -take the selected runtime API probe. +Each model is probed on the API the plan selected (--runtime-api): `converse` +(the default when no API was selected), `responses` / `chat_completions` for the +runtime OpenAI-compatible APIs, or `messages` for the runtime Anthropic Messages +API. Embedding models (which don't speak Converse) go via InvokeModel with their +family's request body. Bare OpenAI proprietary GPT ids are the bedrock-mantle +form and are probed via the OpenAI Responses API on that endpoint; GPT-6 Astra +(us./global.) and GPT-5.6 (us./in./global.) CRIS ids are bedrock-runtime targets +and take the selected runtime API probe. Probe output budgets and cost depend on the selected API. """ import argparse, json, sys @@ -78,8 +80,11 @@ def normalize_api_path(plan_path: str | None, model_ids: list[str], runtime_api: str | None = None) -> str: """Resolve a missing legacy path once for evaluator and rewriter dispatch. - Explicit plans retain their chosen API. Legacy bare proprietary GPT targets - use Mantle Responses; runtime targets retain the existing Converse default. + Explicit plans retain their chosen API. A `runtime_openai_cris` plan must + name the source app's API (`responses` or `chat_completions`; `converse` only + when the plan recorded it explicitly) — it fails closed rather than silently + falling back to Converse. Legacy bare proprietary GPT targets use Mantle + Responses; legacy runtime targets with no path at all retain Converse. A mixed legacy target set needs an explicit plan rather than an unsafe guess. """ if plan_path == "mantle" and model_ids and all(is_mantle_model(mid) for mid in model_ids): @@ -87,9 +92,14 @@ def normalize_api_path(plan_path: str | None, model_ids: list[str], if plan_path == "runtime_openai_cris": paths = {"converse": plan_path, "responses": "runtime_openai_responses", "chat_completions": "runtime_openai_chat"} - if runtime_api is not None and runtime_api not in paths: + if runtime_api is None: + raise ValueError( + "runtime_openai_cris plan has no explicit API — record the source app's API " + "(responses or chat_completions) as required_client_api; Converse is never " + "the implicit default") + if runtime_api not in paths: raise ValueError("Unknown runtime API; resolve the saved migration plan") - return paths[runtime_api or "converse"] + return paths[runtime_api] if plan_path: return plan_path if not model_ids: @@ -138,6 +148,49 @@ def probe_runtime_openai_model(model_id: str, region: str, api_surface: str) -> return classify_invoke_error(code, str(e)) +def create_messages_client(region: str, http_client=None, max_retries: int = 0): + """Anthropic SDK client for the bedrock-runtime Messages endpoint. + + Public basis: https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html + (read 2026-11-07) — base_url `https://bedrock-runtime.{region}.amazonaws.com/anthropic` + and a Bedrock bearer token passed as `api_key`. The pinned Anthropic SDK takes a + STRING credential only, so the token is minted here, once per client, and never + passed as a callable; callers create a client per request boundary and close it + (`with create_messages_client(region) as client:`).""" + import anthropic + from aws_bedrock_token_generator import provide_token + extra = {"http_client": http_client} if http_client is not None else {} + return anthropic.Anthropic( + base_url=f"https://bedrock-runtime.{region}.amazonaws.com/anthropic", + api_key=provide_token(region=region), max_retries=max_retries, **extra, + ) + + +def probe_runtime_messages_model(model_id: str, region: str) -> dict: + """Probe the runtime Anthropic Messages API for the exact validated profile. + + Fails CLOSED when the Anthropic SDK or the token generator is missing, for the + same reason probe_mantle_model does: a missing dependency would otherwise turn + the preflight into an unconditional pass with access never verified.""" + try: + with create_messages_client(region) as client: + client.messages.create(model=model_id, max_tokens=16, + messages=[{"role": "user", "content": "ping"}]) + return {"ok": True, "reason": "ok", "detail": "Runtime Anthropic Messages API authorized."} + except ImportError as e: + return {"ok": False, "reason": "messages_deps_missing", + "detail": f"Cannot verify the runtime Anthropic Messages API for {model_id}: {e}. " + f"This needs 'anthropic' and 'aws-bedrock-token-generator', both " + f"declared in scripts/pyproject.toml — re-sync the pinned environment " + f"(`uv sync --project `). Access was NOT verified; " + f"preflight fails closed rather than assuming it works."} + except Exception as e: # noqa: BLE001 - SDK raises many types; status is what matters + code = {401: "UnrecognizedClientException", 403: "AccessDeniedException", + 404: "ResourceNotFoundException", 429: "ThrottlingException"}.get( + getattr(e, "status_code", None), "unknown") + return classify_invoke_error(code, str(e)) + + def classify_mantle_error(status: int | None, message: str) -> dict: """Pure: map an OpenAI-SDK HTTP status from the mantle endpoint to a verdict. @@ -315,7 +368,8 @@ def main(argv=None) -> int: ap.add_argument("--region", required=True) ap.add_argument("--models", required=True, help="comma-separated model ids") ap.add_argument("--dataset-size", type=int, default=0) - ap.add_argument("--runtime-api", choices=["converse", "responses", "chat_completions"], + ap.add_argument("--runtime-api", + choices=["converse", "responses", "chat_completions", "messages"], default="converse") args = ap.parse_args(argv) @@ -357,7 +411,9 @@ def main(argv=None) -> int: "Do not assume GPT-5.6's cached-input quota exemption or RPM rules " "apply; Astra's documented 10x output burndown is runtime-specific.") else: - if (args.runtime_api != "converse" and ".openai.gpt-" in model_id + if args.runtime_api == "messages": + verdict = probe_runtime_messages_model(model_id, args.region) + elif (args.runtime_api != "converse" and ".openai.gpt-" in model_id and "gpt-oss" not in model_id): verdict = probe_runtime_openai_model(model_id, args.region, args.runtime_api) else: diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/pyproject.toml b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/pyproject.toml index df03ec92e..a1bf1c4ed 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/pyproject.toml +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/pyproject.toml @@ -6,12 +6,16 @@ dependencies = [ "boto3>=1.35,<2", "botocore>=1.35,<2", "jsonschema>=4,<5", - # Required to preflight OpenAI's proprietary GPT models, which are - # bedrock-mantle-only and speak neither Converse nor InvokeModel. Without - # these, probe_mantle_model cannot run and preflight cannot verify - # endpoint/model/IAM access for any openai.gpt-5* target. + # Required to preflight the OpenAI-compatible APIs: bare proprietary GPT ids + # are the bedrock-mantle form (OpenAI Responses/Chat on that endpoint) and + # CRIS ids take the runtime /openai/v1 APIs. Without these, probe_mantle_model + # and probe_runtime_openai_model cannot run and preflight cannot verify + # endpoint/model/IAM access for those targets. "openai>=2.45.0,<3", "aws-bedrock-token-generator>=1,<2", + # Required to preflight the bedrock-runtime Anthropic Messages API + # (probe_runtime_messages_model) for runtime_messages plans. + "anthropic>=0.75,<1", ] [dependency-groups] diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/schemas/eval.json b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/schemas/eval.json index eef6efdab..bc292212d 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/schemas/eval.json +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/schemas/eval.json @@ -38,7 +38,7 @@ "properties": { "reason": { "type": "string", - "enum": ["model_access", "source_key_auth", "model_unresolvable", "authz", "mantle_deps_missing"] + "enum": ["model_access", "source_key_auth", "model_unresolvable", "authz", "mantle_deps_missing", "messages_deps_missing"] }, "detail": { "type": "string" } } diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_astra_execution_contract.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_astra_execution_contract.py index 99212c71d..1d854d8c6 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_astra_execution_contract.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_astra_execution_contract.py @@ -5,7 +5,7 @@ import textwrap from pathlib import Path from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import MagicMock, Mock import pytest @@ -28,22 +28,29 @@ def clients(monkeypatch): chat = Mock(return_value=SimpleNamespace(choices=[ SimpleNamespace(message=SimpleNamespace(content="answer"))])) runtime = Mock(return_value={"output": {"message": {"content": [{"text": "answer"}]}}}) + messages = Mock(return_value=SimpleNamespace( + content=[SimpleNamespace(type="text", text="answer")], stop_reason="end_turn")) client = SimpleNamespace(responses=SimpleNamespace(create=responses), chat=SimpleNamespace(completions=SimpleNamespace(create=chat))) + # The Messages client is used as a context manager (`with create_messages_client(...)`). + anth = MagicMock() + anth.__enter__.return_value = anth + anth.messages.create = messages monkeypatch.setitem(sys.modules, "openai", SimpleNamespace( BedrockOpenAI=lambda **k: client, OpenAI=lambda **k: client)) + monkeypatch.setitem(sys.modules, "anthropic", SimpleNamespace(Anthropic=lambda **k: anth)) monkeypatch.setitem(sys.modules, "aws_bedrock_token_generator", SimpleNamespace(provide_token=lambda **k: "test-token")) import boto3 monkeypatch.setattr(boto3, "client", lambda *a, **k: SimpleNamespace(converse=runtime)) - return responses, chat, runtime + return responses, chat, runtime, messages @pytest.mark.parametrize("surface", ["responses", "chat_completions"]) @pytest.mark.parametrize("mid", ["openai.gpt-6-astra", "us.openai.gpt-6-astra", "global.openai.gpt-6-astra"]) def test_astra_connectivity_uses_selected_openai_api(surface, mid, monkeypatch): - responses, chat, runtime = clients(monkeypatch) + responses, chat, runtime, messages = clients(monkeypatch) code = snippet("# 6a. OpenAI-compatible connectivity", "# 7. Load golden") code = code.replace("", mid).replace("", surface) monkeypatch.setenv("AWS_REGION", "us-west-2") @@ -52,6 +59,7 @@ def test_astra_connectivity_uses_selected_openai_api(surface, mid, monkeypatch): assert selected.call_args.kwargs["model"] == mid assert selected.call_count == 1 runtime.assert_not_called() + messages.assert_not_called() (responses if surface == "chat_completions" else chat).assert_not_called() @@ -66,14 +74,15 @@ def test_astra_connectivity_uses_selected_openai_api(surface, mid, monkeypatch): ("openai.gpt-6-astra", "legacy_mantle", "responses"), ("openai.gpt-5.6-sol", "legacy_mantle", "responses"), ("us.openai.gpt-5.6-sol", "converse", "runtime"), + ("global.anthropic.claude-sonnet-5", "messages", "messages"), ]) def test_golden_loop_preserves_endpoint_images_and_resume(mid, surface, endpoint, monkeypatch, tmp_path): - responses, chat, runtime = clients(monkeypatch) + responses, chat, runtime, messages = clients(monkeypatch) if surface == "legacy_mantle": path = preflight_bedrock.normalize_api_path("mantle", [mid]) surface = ("chat_completions" if path.endswith("_openai_chat") else "responses" if path.endswith("_openai_responses") else "converse") - calls = {"responses": responses, "chat": chat, "runtime": runtime} + calls = {"responses": responses, "chat": chat, "runtime": runtime, "messages": messages} data = tmp_path / ".saws-migrate/golden-dataset" out = tmp_path / ".saws-migrate/eval-results" data.mkdir(parents=True) @@ -98,6 +107,14 @@ def test_golden_loop_preserves_endpoint_images_and_resume(mid, surface, endpoint elif endpoint == "chat": assert call["messages"][0] == {"role": "system", "content": "be concise"} assert call["messages"][1]["content"][1]["image_url"]["url"].startswith("data:image/jpeg;base64,") + elif endpoint == "messages": + # Anthropic Messages shape: base64 `source` block (not a data URL), top-level `system`. + image_block = call["messages"][0]["content"][0] + assert image_block["type"] == "image" + assert image_block["source"]["media_type"] == "image/jpeg" + assert image_block["source"]["type"] == "base64" + assert call["messages"][0]["content"][1] == {"type": "text", "text": "describe"} + assert call["system"] == "be concise" else: assert call["messages"][0]["content"][0]["image"]["format"] == "jpeg" for name, spy in calls.items(): @@ -109,7 +126,7 @@ def test_golden_loop_preserves_endpoint_images_and_resume(mid, surface, endpoint @pytest.mark.parametrize("mid", ["openai.gpt-6-astra", "global.openai.gpt-6-astra"]) @pytest.mark.parametrize("surface", ["responses", "chat_completions"]) def test_golden_openai_throttle_returns_partial_without_scoring_error(mid, surface, monkeypatch, tmp_path): - responses, chat, runtime = clients(monkeypatch) + responses, chat, runtime, messages = clients(monkeypatch) selected = chat if surface == "chat_completions" else responses error = RuntimeError("throttled") error.status_code = 429 @@ -134,7 +151,7 @@ def test_golden_openai_throttle_returns_partial_without_scoring_error(mid, surfa @pytest.mark.parametrize("surface,path", [ - (None, "runtime_openai_cris"), ("converse", "runtime_openai_cris"), + ("converse", "runtime_openai_cris"), ("responses", "runtime_openai_responses"), ("chat_completions", "runtime_openai_chat"), ]) def test_explicit_runtime_api_survives_normalization(surface, path): @@ -144,25 +161,104 @@ def test_explicit_runtime_api_survives_normalization(surface, path): preflight_bedrock.normalize_api_path("runtime_openai_cris", [], "unknown") -@pytest.mark.parametrize("surface", ["responses", "chat_completions"]) -def test_runtime_preflight_uses_selected_api(surface, monkeypatch, capsys): - responses, chat, runtime = clients(monkeypatch) +def test_runtime_openai_cris_without_api_fails_closed(): + """A runtime_openai_cris plan with no recorded API must never resolve to Converse.""" + with pytest.raises(ValueError, match="never the implicit default"): + preflight_bedrock.normalize_api_path("runtime_openai_cris", ["us.openai.gpt-6-astra"], None) + # The same rule holds for the snippet SKILL.md tells the orchestrator to run. + skill = (PLUGIN / "skills/llm-to-bedrock/SKILL.md").read_text() + block = skill.split("### The context block", 1)[1] + code = re.search(r"```python\n(.*?)\n```", block, re.S).group(1) + code = code.replace("", str(Path(__file__).parent)) + ns = {"plan_migration_path": "runtime_openai_cris", + "validated_target_model_ids": ["us.openai.gpt-6-astra"], "plan_runtime_api": None} + with pytest.raises(ValueError, match="never the implicit default"): + exec(compile(code, "SKILL.md context block", "exec"), ns) # nosec B102 - committed snippet + ns["plan_runtime_api"] = "responses" + exec(compile(code, "SKILL.md context block", "exec"), ns) # nosec B102 - committed snippet + assert ns["resolved_api_path"] == "runtime_openai_responses" + assert "retains the Converse default otherwise" not in skill + + +@pytest.mark.parametrize("surface,mid", [ + ("responses", "global.openai.gpt-6-astra"), + ("chat_completions", "global.openai.gpt-6-astra"), + ("messages", "global.anthropic.claude-sonnet-5"), +]) +def test_runtime_preflight_uses_selected_api(surface, mid, monkeypatch, capsys): + responses, chat, runtime, messages = clients(monkeypatch) monkeypatch.setattr(preflight_bedrock, "fetch_bedrock_quotas", lambda region: []) - mid = "global.openai.gpt-6-astra" assert preflight_bedrock.main( ["--region", "us-west-2", "--models", mid, "--runtime-api", surface]) == 0 - selected = chat if surface == "chat_completions" else responses + selected = {"responses": responses, "chat_completions": chat, "messages": messages}[surface] assert selected.call_args.kwargs["model"] == mid - runtime.assert_not_called() + assert selected.call_count == 1 + for spy in (responses, chat, runtime, messages): + if spy is not selected: + spy.assert_not_called() assert json.loads(capsys.readouterr().out)["ok"] +def test_runtime_messages_client_uses_real_sdk_wire_contract(monkeypatch): + """Real Anthropic SDK + offline HTTP: the probe POSTs to the runtime /anthropic endpoint + with the minted token as a plain `x-api-key` string.""" + import httpx + import aws_bedrock_token_generator + requests = [] + + def transport(request): + requests.append(request) + return httpx.Response(200, json={ + "id": "msg_fixture", "type": "message", "role": "assistant", + "model": "global.anthropic.claude-sonnet-5", + "content": [{"type": "text", "text": "answer"}], + "stop_reason": "end_turn", "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}}) + + real_factory = preflight_bedrock.create_messages_client + monkeypatch.setattr(aws_bedrock_token_generator, "provide_token", lambda **k: "fixture-token") + with httpx.Client(transport=httpx.MockTransport(transport)) as http_client: + monkeypatch.setattr( + preflight_bedrock, "create_messages_client", + lambda region, max_retries=0: real_factory( + region, http_client=http_client, max_retries=max_retries)) + assert preflight_bedrock.probe_runtime_messages_model( + "global.anthropic.claude-sonnet-5", "us-west-2")["ok"] + assert len(requests) == 1 + assert str(requests[0].url) == "https://bedrock-runtime.us-west-2.amazonaws.com/anthropic/v1/messages" + assert requests[0].headers["x-api-key"] == "fixture-token" + assert "anthropic-version" in requests[0].headers + body = json.loads(requests[0].content) + assert body["model"] == "global.anthropic.claude-sonnet-5" + assert body["messages"] == [{"role": "user", "content": "ping"}] + + +def test_runtime_messages_token_is_minted_per_client(monkeypatch): + """The pinned Anthropic SDK takes a string credential: each client gets a fresh token + string from provide_token, never a callable.""" + ctor = Mock(return_value=MagicMock()) + provide_token = Mock(side_effect=["token-1", "token-2"]) + monkeypatch.setitem(sys.modules, "anthropic", SimpleNamespace(Anthropic=ctor)) + monkeypatch.setitem(sys.modules, "aws_bedrock_token_generator", + SimpleNamespace(provide_token=provide_token)) + preflight_bedrock.create_messages_client("us-west-2") + preflight_bedrock.create_messages_client("us-west-2") + keys = [call.kwargs["api_key"] for call in ctor.call_args_list] + assert keys == ["token-1", "token-2"] + assert all(isinstance(key, str) for key in keys) + assert provide_token.call_args_list[0].kwargs == {"region": "us-west-2"} + for call in ctor.call_args_list: + assert call.kwargs["base_url"] == "https://bedrock-runtime.us-west-2.amazonaws.com/anthropic" + assert call.kwargs["max_retries"] == 0 + assert "http_client" not in call.kwargs + + @pytest.mark.parametrize("surface", ["responses", "chat_completions"]) @pytest.mark.parametrize("mid", ["openai.gpt-6-astra", "global.openai.gpt-6-astra"]) def test_vision_smoke_preserves_openai_endpoint_and_surface(surface, mid, monkeypatch): import io import urllib.request - responses, chat, runtime = clients(monkeypatch) + responses, chat, runtime, messages = clients(monkeypatch) monkeypatch.setattr(urllib.request, "urlopen", lambda *a, **k: io.BytesIO(b"test-image")) code = snippet("# 9.5a OpenAI-compatible vision", "# 10. Run golden") code = code.replace("", mid).replace("", surface) @@ -221,11 +317,27 @@ def test_dispatch_and_rewriter_preserve_runtime_openai_strategy(): rewriter = (PLUGIN / "agents/llm2bedrock-code-rewriter.md").read_text() assert "validated_target_model_ids, plan_runtime_api" in skill assert 'set `rewrite_strategy = "runtime_openai"`' in skill + assert 'set `rewrite_strategy = "runtime_messages"`' in skill + assert "messages for runtime_messages" in skill assert "Target API surface:" in skill - lane = rewriter.split("### Runtime OpenAI lane", 1)[1].split("### Mantle express lane", 1)[0] + lane = rewriter.split("### Runtime OpenAI lane", 1)[1].split( + "### Runtime Anthropic Messages lane", 1)[0] assert "bedrock-runtime.{region}.amazonaws.com/openai/v1" in lane assert "chat.completions.create" in lane and "responses.create" in lane assert "OpenAI-to-boto3 dependency replacement" in lane + messages_lane = rewriter.split("### Runtime Anthropic Messages lane", 1)[1].split( + "### Mantle express lane", 1)[0] + assert "bedrock-runtime.{region}.amazonaws.com/anthropic" in messages_lane + assert 'base_url=f"https://bedrock-runtime.{REGION}.amazonaws.com/anthropic"' in messages_lane + assert "provide_token(region=" in messages_lane + assert "with bedrock_anthropic() as client" in messages_lane + assert "messages.stream(" in messages_lane + assert "bedrock:GetInferenceProfile" in messages_lane + assert "api_key=lambda" not in messages_lane and "auth_token=lambda" not in messages_lane + # Mantle is selector-gated now; the old 'default and smallest change' claim is gone. + assert "remains the default and smallest change" not in rewriter + assert "### Converse rewrite (default)" not in rewriter + assert "### Converse rewrite (explicit `converse` plans)" in rewriter def test_chat_image_helper_validates_missing_bytes(): @@ -269,7 +381,7 @@ def test_api_handoff_paths_and_real_evaluator_classifier_are_consistent(): (None, ["global.openai.gpt-6-astra"], "converse"), ("mantle_openai_chat", ["openai.gpt-6-astra"], "mantle_openai_chat"), ("mantle_openai_responses", ["openai.gpt-6-astra"], "mantle_openai_responses"), - ("runtime_openai_cris", ["us.openai.gpt-6-astra"], "runtime_openai_cris"), + ("runtime_messages", ["global.anthropic.claude-sonnet-5"], "runtime_messages"), ("mantle_messages", ["anthropic.claude-sonnet-5"], "mantle_messages"), ]) def test_dispatch_and_resume_share_api_default(path, ids, expected): diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py index bd1806ef2..a2293c3de 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py @@ -221,13 +221,37 @@ def test_astra_cris_id_grants_runtime_foundation_and_profile_arns(prefix): mid = f"{prefix}.openai.gpt-6-astra" assert not is_mantle_model(mid) statements = generate_policy([mid], "us-west-2", ACCOUNT)["Statement"] - assert len(statements) == 1 + assert len(statements) == 2 assert statements[0]["Action"] == ["bedrock:InvokeModel", "bedrock:InvokeModelWithResponseStream"] assert set(statements[0]["Resource"]) == { "arn:aws:bedrock:*::foundation-model/openai.gpt-6-astra", f"arn:aws:bedrock:us-west-2:{ACCOUNT}:inference-profile/{mid}", f"arn:aws:bedrock:us-west-2:{ACCOUNT}:project/default", } + # Invoking through a CRIS profile also needs GetInferenceProfile on the profile only + # (inference-profiles-prereq.html, read 2026-11-07). + assert statements[1]["Sid"] == "BedrockGetInferenceProfile" + assert statements[1]["Action"] == ["bedrock:GetInferenceProfile"] + assert statements[1]["Resource"] == [f"arn:aws:bedrock:us-west-2:{ACCOUNT}:inference-profile/{mid}"] + + +def test_runtime_messages_profile_gets_invoke_and_get_inference_profile(): + # A runtime_messages target (Anthropic SDK on bedrock-runtime /anthropic) is a plain + # CRIS profile for IAM: InvokeModel* on profile + foundation model, GetInferenceProfile + # on the profile, and no Mantle statement. + mid = "global.anthropic.claude-sonnet-5" + statements = generate_policy([mid], "us-west-2", ACCOUNT)["Statement"] + assert [s["Sid"] for s in statements] == ["BedrockInvokeModelScoped", "BedrockGetInferenceProfile"] + assert set(statements[0]["Resource"]) == { + "arn:aws:bedrock:*::foundation-model/anthropic.claude-sonnet-5", + f"arn:aws:bedrock:us-west-2:{ACCOUNT}:inference-profile/{mid}", + } + assert statements[1]["Resource"] == [f"arn:aws:bedrock:us-west-2:{ACCOUNT}:inference-profile/{mid}"] + + +def test_bare_foundation_model_emits_no_get_inference_profile(): + statements = generate_policy(["anthropic.claude-haiku-4-5-20251001-v1:0"], REGION, ACCOUNT)["Statement"] + assert [s["Sid"] for s in statements] == ["BedrockInvokeModelScoped"] class TestGpt6CrisProjectArn: @@ -317,7 +341,8 @@ def test_predicates_exclude_bare_mantle_gpt56_and_unrelated_ids(self, model_id): @pytest.mark.parametrize("model_id", _GPT54_55_CRIS_SHORT_IDS) def test_generate_policy_grants_project_default_for_short_id(self, model_id): pol = generate_policy([model_id], REGION, ACCOUNT) - assert [s["Sid"] for s in pol["Statement"]] == ["BedrockInvokeModelScoped"] + # A CRIS profile also carries the GetInferenceProfile statement (inference-profiles-prereq.html). + assert [s["Sid"] for s in pol["Statement"]] == ["BedrockInvokeModelScoped", "BedrockGetInferenceProfile"] base = model_id.split(".", 1)[1] assert set(pol["Statement"][0]["Resource"]) == { f"arn:aws:bedrock:*::foundation-model/{base}", diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_image_input.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_image_input.py index 127077ae0..8b89f9efe 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_image_input.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_image_input.py @@ -76,12 +76,30 @@ def test_converse_message_text_only(): "role": "user", "content": [{"text": "describe"}]} -@pytest.mark.parametrize("fn", [ii.converse_message, ii.responses_message]) +@pytest.mark.parametrize("fn", [ii.converse_message, ii.responses_message, ii.chat_message, + ii.messages_message]) def test_image_path_without_bytes_is_an_error(fn): with pytest.raises(ValueError, match="raw image bytes"): fn("describe", "cat.jpg", None) +def test_messages_message_uses_a_base64_source_block_before_the_text(): + # Anthropic Messages takes a base64 `source` block with a normalized media_type + # (`.jpg` -> `image/jpeg`), not an OpenAI-style data URL. + msg = ii.messages_message("describe", "cat.jpg", RAW) + assert msg["role"] == "user" + assert [b["type"] for b in msg["content"]] == ["image", "text"] + source = msg["content"][0]["source"] + assert source == {"type": "base64", "media_type": "image/jpeg", + "data": base64.b64encode(RAW).decode()} + assert msg["content"][1] == {"type": "text", "text": "describe"} + + +def test_messages_message_text_only(): + assert ii.messages_message("describe") == { + "role": "user", "content": [{"type": "text", "text": "describe"}]} + + def test_responses_image_block_sets_detail(): # Regression: `detail` is Required on ResponseInputImageParam in the pinned SDK, # and omitting it is a contract violation a TypedDict will not catch at runtime. diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py index 3056e8b0a..4d56f91a6 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py @@ -322,3 +322,51 @@ def test_mantle_deps_are_declared_in_pinned_env(): toml = (pathlib.Path(__file__).resolve().parent / "pyproject.toml").read_text() assert "openai>=2.45.0" in toml assert "aws-bedrock-token-generator" in toml + assert "anthropic>=" in toml + + +def test_messages_missing_deps_fails_closed(monkeypatch): + # Same contract as the mantle probe: a missing Anthropic SDK must not turn the + # runtime Messages preflight into an unconditional pass. + import builtins + real_import = builtins.__import__ + + def no_anthropic(name, *a, **k): + if name in ("anthropic", "aws_bedrock_token_generator"): + raise ImportError(f"No module named {name!r}") + return real_import(name, *a, **k) + + monkeypatch.setattr(builtins, "__import__", no_anthropic) + v = p.probe_runtime_messages_model("global.anthropic.claude-sonnet-5", "us-west-2") + assert v["ok"] is False + assert v["reason"] == "messages_deps_missing" + assert "NOT verified" in v["detail"] + + +def test_runtime_openai_cris_plan_without_api_fails_closed(): + # Regression: a runtime_openai_cris plan with no recorded API used to resolve to + # Converse silently; the source app's API must be recorded (or explicit converse). + with pytest.raises(ValueError, match="never the implicit default"): + p.normalize_api_path("runtime_openai_cris", ["us.openai.gpt-5.5"], None) + assert p.normalize_api_path("runtime_openai_cris", ["us.openai.gpt-5.5"], "converse") == "runtime_openai_cris" + assert p.normalize_api_path("runtime_messages", ["global.anthropic.claude-sonnet-5"]) == "runtime_messages" + + +def test_main_routes_messages_api_to_the_messages_probe(monkeypatch, capsys): + import boto3 + import json + calls = [] + monkeypatch.setattr(boto3, "client", lambda *a, **k: object()) + monkeypatch.setattr(p, "fetch_bedrock_quotas", lambda region: []) + monkeypatch.setattr(p, "probe_runtime_messages_model", + lambda mid, region: calls.append(("messages", mid, region)) or + {"ok": True, "reason": "ok", "detail": "ok"}) + monkeypatch.setattr(p, "probe_model", + lambda client, mid: (_ for _ in ()).throw(AssertionError("Converse probe must not run"))) + monkeypatch.setattr(p, "probe_runtime_openai_model", + lambda *a: (_ for _ in ()).throw(AssertionError("OpenAI probe must not run"))) + rc = p.main(["--region", "us-west-2", "--models", "global.anthropic.claude-sonnet-5", + "--runtime-api", "messages"]) + assert rc == 0 + assert calls == [("messages", "global.anthropic.claude-sonnet-5", "us-west-2")] + assert json.loads(capsys.readouterr().out)["ok"] is True diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/uv.lock b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/uv.lock index dbb05e859..8288e7f75 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/uv.lock +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/uv.lock @@ -11,6 +11,7 @@ name = "ai-to-bedrock-scripts" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "anthropic" }, { name = "aws-bedrock-token-generator" }, { name = "boto3" }, { name = "botocore" }, @@ -25,6 +26,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "anthropic", specifier = ">=0.75,<1" }, { name = "aws-bedrock-token-generator", specifier = ">=1,<2" }, { name = "boto3", specifier = ">=1.35,<2" }, { name = "botocore", specifier = ">=1.35,<2" }, @@ -44,6 +46,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, ] +[[package]] +name = "anthropic" +version = "0.125.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "docstring-parser" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/f8/6f0560884b5363848347bd640b6c1d04abc25e7aa61787a232f790c6b60a/anthropic-0.125.0.tar.gz", hash = "sha256:e0cdd336580cb7411c1cdab69f80973e9bf4bff7f8e08141811d46307d45c682", size = 1112593, upload-time = "2026-08-19T22:00:42.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/1a/b1bd30cda3790557e8791bec5922a6ec8fabb6fa8b008c76a39cf7be6152/anthropic-0.125.0-py3-none-any.whl", hash = "sha256:3486013602eca76d8b12540764e53654f02cf4951110bca86cf06e67428a9f21", size = 1184067, upload-time = "2026-08-19T22:00:44.596Z" }, +] + [[package]] name = "anyio" version = "4.14.2" @@ -134,12 +155,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] +[[package]] +name = "docstring-parser" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, +] + [[package]] name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-anthropic-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-anthropic-to-bedrock.md index 59d6a0796..05cbdf73d 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-anthropic-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-anthropic-to-bedrock.md @@ -89,8 +89,8 @@ Response parsing keeps the native shape: select text blocks by `type == "text"`, position, because thinking and tool-use blocks can appear first. ```python -text = "".join(block.text for block in response.content if block.type == "text") -if not text or response.stop_reason in ("max_tokens", "refusal", "tool_use"): +answer = "".join(block.text for block in response.content if block.type == "text") +if not answer or response.stop_reason in ("max_tokens", "refusal", "tool_use"): raise ValueError(f"No complete text response (stop_reason={response.stop_reason})") ``` @@ -154,15 +154,16 @@ metadata). This is a rewrite, not a repoint: Content becomes typed blocks `[{"text": "..."}]`; `max_tokens` moves to `inferenceConfig.maxTokens`; select text blocks by their `text` key because reasoning and tool blocks can appear first: - ```python - text = "".join(block["text"] for block in response["output"]["message"]["content"] if "text" in block) - stop_reason = response.get("stopReason", "unknown") - if not text or stop_reason in ("max_tokens", "model_context_window_exceeded", "guardrail_intervened", "content_filtered", "refusal", "tool_use"): - raise ValueError(f"No complete text response (stopReason={stop_reason})") - ``` +```python +text = "".join(block["text"] for block in response["output"]["message"]["content"] if "text" in block) +stop_reason = response.get("stopReason", "unknown") +if not text or stop_reason in ("max_tokens", "model_context_window_exceeded", "guardrail_intervened", "content_filtered", "refusal", "tool_use"): + raise ValueError(f"No complete text response (stopReason={stop_reason})") +``` + +For tool loops on Converse, retain the original `response["output"]["message"]` in assistant +history, including signed reasoning blocks, and handle every tool request before continuing. - For tool loops, retain the original `response["output"]["message"]` in assistant history, - including signed reasoning blocks, and handle every tool request before continuing. - System prompts: `system="You are helpful"` → `system=[{"text": "You are helpful"}]`. - Streaming: `client.messages.stream()` → `client.converse_stream()`; iterate `response["stream"]` for `contentBlockDelta` events. diff --git a/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py b/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py index 0ab6ec546..45c38073e 100644 --- a/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py +++ b/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py @@ -7,8 +7,7 @@ `skills/shared/ai/bedrock-endpoint-policy.md`, vendors it into gcp-to-aws and azure-to-aws, and requires every `mantle_*` design row to record a `mantle_selector`. These checks pin the policy file's contract and the design -files that must point at it. The llm-to-bedrock SKILL.md reference is added by -the execution-lane change and is asserted there, not here. +files and the llm-to-bedrock orchestrator that must point at it. """ from __future__ import annotations @@ -24,6 +23,7 @@ ] ANTHROPIC_GUIDE = PLUGIN_ROOT / "skills" / "shared" / "ai" / "ai-anthropic-to-bedrock.md" +LLM_TO_BEDROCK_SKILL = PLUGIN_ROOT / "skills" / "llm-to-bedrock" / "SKILL.md" DESIGN_AI_FILES = [ PLUGIN_ROOT / "skills" / "gcp-to-aws" / "references" / "phases" / "design" / "design-ai.md", @@ -88,6 +88,15 @@ def test_design_ai_files_reference_the_policy() -> None: assert "mantle_selector" in text, f"{f}: does not require mantle_selector on mantle_* rows" +def test_llm_to_bedrock_skill_references_the_policy_and_defaults_to_runtime() -> None: + text = LLM_TO_BEDROCK_SKILL.read_text(encoding="utf-8") + assert "bedrock-endpoint-policy.md" in text, "llm-to-bedrock SKILL.md does not point at the endpoint policy" + assert "## Endpoint policy (bedrock-runtime default)" in text + assert "API continuity alone never selects Mantle" in _norm(text) + assert "retains the Converse default otherwise" not in text + assert "never the implicit default" in _norm(text) + + def test_schema_docs_carry_the_runtime_default_vocabulary() -> None: for f in SCHEMA_FILES: text = f.read_text(encoding="utf-8") From 42cb7a1dbd2f81cbda6dac683a806668c7b8e25c Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 7 Nov 2026 17:43:11 -0800 Subject: [PATCH 11/41] fix(aws-startup-advisor): probe embedding ids via InvokeModel under a runtime_messages plan and align the Mantle Messages snippet with the public page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review iteration 1 findings (review.json CHANGES_REQUESTED): - major: preflight_bedrock.main() routed every non-Mantle id to the Messages probe when --runtime-api messages was set, so a runtime_messages plan whose $TARGET_MODELS also carried an embedding id (B1 collects every bedrock_models[].aws_model_id) failed preflight spuriously. Embedding ids now fall through to probe_model (InvokeModel) before the messages branch. New test_main_messages_plan_probes_embedding_ids_via_invoke_model covers a Claude profile + amazon.titan-embed-text-v2:0 list. - minor: agents/llm2bedrock-code-rewriter.md Mantle express lane Anthropic row and snippet used base_url .../anthropic/v1 with auth_token=; the SDK appends /v1/messages, so that resolved to /anthropic/v1/v1/messages. Now base_url .../anthropic with api_key=, matching the public Messages API page (inference-messages-api.html, read 2026-11-07) and the shared guide. - minor: evaluator §6b and §9.5b heredocs are now executed by test_runtime_messages_connectivity_snippet_uses_the_messages_client and test_runtime_messages_vision_snippet_sends_a_base64_source_block. - minor: preflight_bedrock.py module docstring now scopes in. to GPT-5.6 Terra and Luna, with GPT-5.6 Sol and GPT-6 Astra us./global. only. - minor: the mantle_selector enum paraphrase is removed from skills/shared/ai/ai-openai-to-bedrock.md and gcp design-ai.md; both now point at references/vendored/ai/bedrock-endpoint-policy.md. The llm-to-bedrock SKILL.md section stays as the one permitted restatement. Vendored copies re-synced (tools/sync-vendored.py). Verification (plugins/aws-startup-advisor, 2026-11-07): - sync-vendored --check: PASS 64 (2 re-synced) - llm-to-bedrock scripts: 435 passed (431 + 4 new) - agent-advisor scripts: 570 passed; no non-vendored agent-advisor change - mise run ci: exit 0 (lint:md 0 errors, lint:skill-refs 1520/0 missing, lint:vendored-parity PASS 64, lint:artifacts 99/0, test 1408 passed + 222 subtests, test:telemetry 311, test:heroku-source-review OK) - root python3 tools/validate.py: All validations passed; mise run validate: exit 0 - residual Mantle-default grep over plugins/aws-startup-advisor *.md/*.json: nothing - gitleaks protect --staged: no leaks found --- .../agents/llm2bedrock-code-rewriter.md | 9 ++-- .../vendored/ai/ai-openai-to-bedrock.md | 5 +-- .../references/phases/design/design-ai.md | 3 +- .../vendored/ai/ai-openai-to-bedrock.md | 5 +-- .../scripts/preflight_bedrock.py | 9 ++-- .../scripts/test_astra_execution_contract.py | 42 +++++++++++++++++++ .../scripts/test_preflight_bedrock.py | 27 ++++++++++++ .../skills/shared/ai/ai-openai-to-bedrock.md | 5 +-- 8 files changed, 88 insertions(+), 17 deletions(-) diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md b/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md index bfbd08cff..fe05d5eac 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md @@ -241,7 +241,7 @@ The source SDK stays. Per client, change only three things: | ---------------------------------------- | ------------------------------------------------------ | | Proprietary OpenAI GPT (`openai.gpt-5*` / `openai.gpt-6*`) | `https://bedrock-mantle..api.aws/openai/v1` | | Other OpenAI-compatible targets | `https://bedrock-mantle..api.aws/v1` | - | Anthropic SDK | `https://bedrock-mantle..api.aws/anthropic/v1` | + | Anthropic SDK | `https://bedrock-mantle..api.aws/anthropic` (the SDK appends `/v1/messages`) | If your context has a `Mantle base path` line, use it verbatim — it is authoritative over this table. - **Credential** → a Bedrock bearer token, NOT the original provider key. For a short CLI or one-shot script, read it from the `AWS_BEARER_TOKEN_BEDROCK` env var; for any long-running process use the auto-refreshing `provide_token` client shown below (the env-var form expires within 12 hours). Do not leave the old `api_key=os.environ["OPENAI_API_KEY"]` line in place. @@ -305,9 +305,12 @@ client = anthropic.Anthropic() import os import anthropic client = anthropic.Anthropic( - base_url="https://bedrock-mantle.us-east-1.api.aws/anthropic/v1", - auth_token=os.environ["AWS_BEARER_TOKEN_BEDROCK"], + base_url="https://bedrock-mantle.us-east-1.api.aws/anthropic", + api_key=os.environ["AWS_BEARER_TOKEN_BEDROCK"], ) +# Same base_url/api_key form as the public Messages API page +# (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html, read 2026-11-07); +# the SDK appends /v1/messages and the anthropic-version header itself. ``` Preserve compatible SDK request/response shapes. A model version change still requires §9 target checks, including typed response parsing for Opus 5.5. Apply the Chat Completions → Responses reshape only when that API change was selected. Preserve the exact validated runtime profile ID or ARN. After applying the changes above, skip the Converse-specific guidance in the rest of §8; the §9 behavior-delta application still applies normally. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index 852c8e2fb..e7ce5f31e 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -93,9 +93,8 @@ Amazon Bedrock recommends `bedrock-runtime` for new applications (`references/vendored/ai/bedrock-endpoint-policy.md`). Record `migration_path: "mantle_openai_chat"` (Chat Completions source on a model whose card lists it on mantle — GPT-6, GPT-5.5, GPT-5.4) or `"mantle_openai_responses"` **only** when the row also records a `bedrock_models[].mantle_selector` -from that policy file (background inference, server-side tools / Web Search, Projects / Workspaces, -a Mantle-only model or API, or strict in-Region processing where runtime offers only CRIS). API -continuity alone never selects Mantle. +from `references/vendored/ai/bedrock-endpoint-policy.md` (the selector enum is defined there and +only there). API continuity alone never selects Mantle. Report the assessment honestly: `strong_migrate` here means "low-risk, well-supported move", not "cheaper". diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index b9ede2a5a..4a2e6fe9f 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -411,8 +411,7 @@ ID becomes the CRIS profile and IAM is `bedrock:InvokeModel*`. A `mantle_*` path (`"mantle_openai_chat"` for a Chat source, `"mantle_openai_responses"` for a Responses source — e.g. Astra on Standard Mantle in `us-east-1` or `us-west-2`) is recorded **only** when the row also records `bedrock_models[].mantle_selector` from `references/vendored/ai/bedrock-endpoint-policy.md` -(background inference, server-side tools / Web Search, Projects / Workspaces, a Mantle-only model or -API, or strict in-Region processing where runtime offers only CRIS); then keep the OpenAI SDK, use +(the selector enum is defined there and only there); then keep the OpenAI SDK, use `/openai/v1` on Mantle, Bedrock credentials and Mantle IAM. API continuity alone never selects Mantle. Do not reshape Astra Chat solely because it is proprietary GPT. Record `model_change: false` only when the exact source model is retained; a Pro-to-Astra upgrade is a model change requiring eval. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index 852c8e2fb..e7ce5f31e 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -93,9 +93,8 @@ Amazon Bedrock recommends `bedrock-runtime` for new applications (`references/vendored/ai/bedrock-endpoint-policy.md`). Record `migration_path: "mantle_openai_chat"` (Chat Completions source on a model whose card lists it on mantle — GPT-6, GPT-5.5, GPT-5.4) or `"mantle_openai_responses"` **only** when the row also records a `bedrock_models[].mantle_selector` -from that policy file (background inference, server-side tools / Web Search, Projects / Workspaces, -a Mantle-only model or API, or strict in-Region processing where runtime offers only CRIS). API -continuity alone never selects Mantle. +from `references/vendored/ai/bedrock-endpoint-policy.md` (the selector enum is defined there and +only there). API continuity alone never selects Mantle. Report the assessment honestly: `strong_migrate` here means "low-risk, well-supported move", not "cheaper". diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py index f2fd9d059..1b4c31b79 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py @@ -12,8 +12,9 @@ API. Embedding models (which don't speak Converse) go via InvokeModel with their family's request body. Bare OpenAI proprietary GPT ids are the bedrock-mantle form and are probed via the OpenAI Responses API on that endpoint; GPT-6 Astra -(us./global.) and GPT-5.6 (us./in./global.) CRIS ids are bedrock-runtime targets -and take the selected runtime API probe. +(us./global.), GPT-5.6 Terra and Luna (us./in./global.) and GPT-5.6 Sol +(us./global.) CRIS ids are bedrock-runtime targets and take the selected +runtime API probe. Probe output budgets and cost depend on the selected API. """ import argparse, json, sys @@ -411,7 +412,9 @@ def main(argv=None) -> int: "Do not assume GPT-5.6's cached-input quota exemption or RPM rules " "apply; Astra's documented 10x output burndown is runtime-specific.") else: - if args.runtime_api == "messages": + # Embedding ids in a chat plan's $TARGET_MODELS never speak the selected + # chat API; they are always probed via InvokeModel (probe_model). + if args.runtime_api == "messages" and not is_embedding_model(model_id): verdict = probe_runtime_messages_model(model_id, args.region) elif (args.runtime_api != "converse" and ".openai.gpt-" in model_id and "gpt-oss" not in model_id): diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_astra_execution_contract.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_astra_execution_contract.py index 1d854d8c6..3da02751f 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_astra_execution_contract.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_astra_execution_contract.py @@ -1,4 +1,5 @@ """Execute the actual evaluator snippets and validate Astra handoff contracts.""" +import base64 import json import re import sys @@ -63,6 +64,47 @@ def test_astra_connectivity_uses_selected_openai_api(surface, mid, monkeypatch): (responses if surface == "chat_completions" else chat).assert_not_called() +@pytest.mark.parametrize("mid", ["global.anthropic.claude-sonnet-5", "us.anthropic.claude-opus-5.5"]) +def test_runtime_messages_connectivity_snippet_uses_the_messages_client(mid, monkeypatch, capsys): + # §6b is the Messages counterpart of §6a; execute the committed heredoc so a typo + # in the evaluator prose cannot ship unnoticed. + responses, chat, runtime, messages = clients(monkeypatch) + code = snippet("# 6b. Runtime Anthropic Messages connectivity", "# 7. Load golden") + code = code.replace("", mid).replace("", str(Path(__file__).parent)) + monkeypatch.setenv("AWS_REGION", "us-west-2") + exec(compile(code, str(EVALUATOR), "exec"), {}) # nosec B102 - committed test fixture + assert messages.call_count == 1 + assert messages.call_args.kwargs["model"] == mid + assert messages.call_args.kwargs["messages"] == [{"role": "user", "content": "ping"}] + assert "OK: answer" in capsys.readouterr().out + responses.assert_not_called() + chat.assert_not_called() + runtime.assert_not_called() + + +def test_runtime_messages_vision_snippet_sends_a_base64_source_block(monkeypatch, capsys): + import io + import urllib.request + responses, chat, runtime, messages = clients(monkeypatch) + monkeypatch.setattr(urllib.request, "urlopen", lambda *a, **k: io.BytesIO(b"test-image")) + code = snippet("# 9.5b Runtime Anthropic Messages vision", "# 10. Run golden") + code = code.replace("", "global.anthropic.claude-sonnet-5") + code = code.replace("", str(Path(__file__).parent)) + monkeypatch.setenv("AWS_REGION", "us-west-2") + exec(compile(code, str(EVALUATOR), "exec"), {}) # nosec B102 - committed smoke snippet + call = messages.call_args.kwargs + assert call["model"] == "global.anthropic.claude-sonnet-5" + image_block, text_block = call["messages"][0]["content"] + assert image_block["type"] == "image" + assert image_block["source"] == {"type": "base64", "media_type": "image/jpeg", + "data": base64.b64encode(b"test-image").decode()} + assert text_block == {"type": "text", "text": "Describe this image briefly."} + assert "VISION_OK: answer" in capsys.readouterr().out + responses.assert_not_called() + chat.assert_not_called() + runtime.assert_not_called() + + @pytest.mark.parametrize("mid,surface,endpoint", [ ("openai.gpt-6-astra", "responses", "responses"), ("openai.gpt-6-astra", "chat_completions", "chat"), diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py index 4d56f91a6..5fd546889 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py @@ -370,3 +370,30 @@ def test_main_routes_messages_api_to_the_messages_probe(monkeypatch, capsys): assert rc == 0 assert calls == [("messages", "global.anthropic.claude-sonnet-5", "us-west-2")] assert json.loads(capsys.readouterr().out)["ok"] is True + + +def test_main_messages_plan_probes_embedding_ids_via_invoke_model(monkeypatch, capsys): + # Regression: B1 collects every bedrock_models[].aws_model_id into $TARGET_MODELS and + # B4 passes the whole list with one --runtime-api. A runtime_messages plan with a + # Claude profile plus an embedding id POSTed the embedding id to /anthropic/v1/messages + # and failed preflight spuriously; embeddings must still go via InvokeModel. + import boto3 + import json + calls = [] + monkeypatch.setattr(boto3, "client", lambda *a, **k: object()) + monkeypatch.setattr(p, "fetch_bedrock_quotas", lambda region: []) + monkeypatch.setattr(p, "probe_runtime_messages_model", + lambda mid, region: calls.append(("messages", mid)) or + {"ok": True, "reason": "ok", "detail": "ok"}) + monkeypatch.setattr(p, "probe_model", + lambda client, mid: calls.append(("invoke_model", mid)) or + {"ok": True, "reason": "ok", "detail": "ok"}) + monkeypatch.setattr(p, "probe_runtime_openai_model", + lambda *a: (_ for _ in ()).throw(AssertionError("OpenAI probe must not run"))) + rc = p.main(["--region", "us-west-2", + "--models", "global.anthropic.claude-sonnet-5,amazon.titan-embed-text-v2:0", + "--runtime-api", "messages"]) + assert rc == 0 + assert calls == [("messages", "global.anthropic.claude-sonnet-5"), + ("invoke_model", "amazon.titan-embed-text-v2:0")] + assert json.loads(capsys.readouterr().out)["ok"] is True diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md index 852c8e2fb..e7ce5f31e 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md @@ -93,9 +93,8 @@ Amazon Bedrock recommends `bedrock-runtime` for new applications (`references/vendored/ai/bedrock-endpoint-policy.md`). Record `migration_path: "mantle_openai_chat"` (Chat Completions source on a model whose card lists it on mantle — GPT-6, GPT-5.5, GPT-5.4) or `"mantle_openai_responses"` **only** when the row also records a `bedrock_models[].mantle_selector` -from that policy file (background inference, server-side tools / Web Search, Projects / Workspaces, -a Mantle-only model or API, or strict in-Region processing where runtime offers only CRIS). API -continuity alone never selects Mantle. +from `references/vendored/ai/bedrock-endpoint-policy.md` (the selector enum is defined there and +only there). API continuity alone never selects Mantle. Report the assessment honestly: `strong_migrate` here means "low-risk, well-supported move", not "cheaper". From fee5ffc8cc6056aeff16121afe5c51bf12f3c240 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 7 Nov 2026 19:47:59 -0800 Subject: [PATCH 12/41] =?UTF-8?q?fix(aws-startup-advisor):=20address=20PR?= =?UTF-8?q?=20585=20review=20round=201=20=E2=80=94=20default-project=20gra?= =?UTF-8?q?nt=20by=20id=20form/API,=20per-row=20preflight=20APIs,=20mantle?= =?UTF-8?q?=5Fmessages=20lane=20gate,=20resolved=5Fruntime=5Fapi=20in=20re?= =?UTF-8?q?sume=20context?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-checked each leon1418 finding against 6ca403b6 (post-rebase onto 8c37ef0f): 1. Default-project grant (ai-openai-to-bedrock.md:339). Partly satisfied by the rebase: 8c37ef0f grants project/default for GPT-6 and the GPT-5.4/5.5 CRIS ids, but GPT-5.6 Sol/Terra/Luna were still excluded and the grant was model-family-based. The public Responses API page (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, "Permissions", read 2026-11-07) authorizes bedrock:InvokeModel on the default project for every Responses request on bedrock-runtime, and the GPT-5.6 Sol/Terra/Luna cards (read 2026-11-07) repeat it for their runtime CRIS ids (Chat tab included). iam_policy.py now classifies by id form (is_runtime_openai_cris_model: us./in./us-gov./global. openai.gpt-* ids and their inference-profile ARNs; _GEO_PREFIX gains in. and us-gov.) and decides the grant by the selected API (--runtime-api responses|chat_completions emit it, converse/messages do not, id-only keeps it). GCP generate-ai.md / generate-artifacts-ai.md, Azure generate-artifacts-ai.md, both openai-on-bedrock.md fact bases, the shared guide (+vendored copies) and the report-generator §6.4 recipe now state the rule and cite the page. 2. Mixed-model preflight (preflight_bedrock.py:417). Confirmed open: one --runtime-api was applied to every id. preflight gains --target ID=API pairs (per-row required_client_api; invoke for embeddings) and fails closed with reason unknown_runtime_api before any probe when a row's API is missing or unknown; --models/--runtime-api stays as the legacy single-API form. B1/B4 pass each row as its own pair. 3. mantle_messages stop (SKILL.md:936). Confirmed open. New lane gate `preflight_bedrock.py --check-lanes aws-design-ai.json` (unsupported_lane_stop) halts before B2/B4/Phase C with a structured unsupported_lane result naming the deferred Mantle Messages lane and the workspaces selector; iam_policy --migration-path mantle_messages refuses to emit runtime permissions for the bare Claude id; the context rule no longer labels it converse. GCP generate-artifacts-ai.md Step 0: the broad Mantle branch is scoped to the three OpenAI-SDK spellings so mantle_messages matches exactly one branch and runtime_messages keeps Messages. The lane is not implemented. 4. resolved_runtime_api (SKILL.md:913). Confirmed open. The field is now in the C0 current-context field list (always present), resolved in B4 before preflight (read back from run-context.json on resume), and has an invalidation row (EVAL, REWRITE, REPORT archived; ANALYSIS, INGESTION kept). Rebase-review non-blocking items: reworded the "keeps the `responses` default" checklist sentences (gcp design-ai.md, azure shared schema-design-aws-ai.md) to the source-API default rule; removed the EOF blank line in tests/test_ai_only_option_c_framework_route.py. Tests: test_iam_policy.py (id-form x API matrix incl. in./us-gov., Converse-only and Anthropic/oss controls, CLI flags, mantle_messages refusal), test_preflight_bedrock.py (offline fake SDK clients: Claude/Messages + GPT/Chat, Claude/Messages + Nova/Converse + embeddings, fail-closed unknown API, --check-lanes), test_validate_result.py (actual comparator: unchanged resume matches, dropped field reproduces the reviewer's MISMATCH, changed API named), tests/test_runtime_default_review_round1.py (text contracts + comparator). Existing GPT-5.6 "no grant" assertions in test_iam_policy.py were flipped to match the 2026-11-07 card read. Validation (worktree fix-llm-to-bedrock-runtime-default): - mise -C plugins/aws-startup-advisor run ci: EXIT 0 (lint:md 0 errors; vendored-parity 64 PASS; skill-refs 1521 PASS; artifacts 99 PASS; test 1429 passed + 222 subtests; test:telemetry 311 passed; heroku 44 pass) - llm-to-bedrock scripts pytest: 599 passed - agent-advisor scripts pytest: 570 passed - python3 tools/validate.py: All validations passed - tools/sync-vendored.py --check: PASS 64 byte-identical - git diff --check: clean - mise run build (gitleaks): fails ONLY on pre-existing upstream f8d09fd1 (tests/test_bedrock_migration_assessment_discover.py:118 generic-api-key, leaks found: 1) — not introduced by this branch --- .../agents/llm2bedrock-report-generator.md | 10 +- .../phases/generate/generate-artifacts-ai.md | 24 +- .../references/shared/openai-on-bedrock.md | 10 +- .../references/shared/schema-design-aws-ai.md | 4 +- .../vendored/ai/ai-openai-to-bedrock.md | 12 +- .../references/phases/design/design-ai.md | 4 +- .../references/phases/generate/generate-ai.md | 2 +- .../phases/generate/generate-artifacts-ai.md | 6 +- .../references/shared/openai-on-bedrock.md | 10 +- .../vendored/ai/ai-openai-to-bedrock.md | 12 +- .../skills/llm-to-bedrock/SKILL.md | 95 +++++-- .../llm-to-bedrock/scripts/iam_policy.py | 120 ++++++-- .../scripts/preflight_bedrock.py | 255 +++++++++++++---- .../llm-to-bedrock/scripts/test_iam_policy.py | 170 ++++++++++- .../scripts/test_preflight_bedrock.py | 217 ++++++++++++++ .../scripts/test_validate_result.py | 26 ++ .../skills/shared/ai/ai-openai-to-bedrock.md | 12 +- .../test_ai_only_option_c_framework_route.py | 1 - .../test_runtime_default_review_round1.py | 266 ++++++++++++++++++ 19 files changed, 1125 insertions(+), 131 deletions(-) create mode 100644 plugins/aws-startup-advisor/tests/test_runtime_default_review_round1.py diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md index 8ca2d0bdb..119f7f97a 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md @@ -568,12 +568,18 @@ uv run --project python /iam_policy.py \ --models \ --region \ --account-id \ + --runtime-api \ + --migration-path \ --output /.saws-migrate/iam-policy.json ``` -The script handles the dual-ARN pattern: foundation-model ARNs for plain model IDs and inference-profile ARNs for geo-prefixed IDs (e.g. `us.anthropic.claude-sonnet-4-6`). For any cross-region (CRIS) target it also emits a `BedrockGetInferenceProfile` statement (`bedrock:GetInferenceProfile` on the profile ARNs) — the documented prerequisite for invoking through a profile (, read 2026-11-07). For a GPT-6 CRIS id and for the GPT-5.5 / GPT-5.4 CRIS ids (`us.`/`global.openai.gpt-5.5`, `us.`/`global.openai.gpt-5.4`, or their `inference-profile/` ARNs) it adds a third resource, `arn:aws:bedrock:::project/default`, because those model cards (GPT-5.4 / GPT-5.5 read 2026-11-07) require `bedrock:InvokeModel` on the default project in addition to the inference profile; do not strip it from the report's policy excerpt. The output is a ready-to-use IAM policy JSON file. +Omit `--runtime-api` only when the context carries no API surface (id-only behaviour). A `mantle_messages` +path never reaches this agent (the orchestrator's lane gate stops it before Phase C); if it somehow does, the +helper exits 1 with `reason: unsupported_lane` — report that verbatim and do not hand-write a policy. -**Runtime OpenAI and Runtime Messages lanes need no Mantle statement.** A `Rewrite strategy: runtime_openai` or `Rewrite strategy: runtime_messages` run keeps the source SDK but authorizes through `bedrock:InvokeModel*` (+ `GetInferenceProfile`, and the default-project ARN for the GPT-6 / GPT-5.5 / GPT-5.4 CRIS ids above) on the `bedrock-runtime` endpoint; report exactly the statements the script produced and do not add `bedrock-mantle:*` for them. Per-token model pricing is the same on both endpoints (, read 2026-11-07), so the endpoint choice does not change the §6.3 cost table. +The script handles the dual-ARN pattern: foundation-model ARNs for plain model IDs and inference-profile ARNs for geo-prefixed IDs (e.g. `us.anthropic.claude-sonnet-4-6`). For any cross-region (CRIS) target it also emits a `BedrockGetInferenceProfile` statement (`bedrock:GetInferenceProfile` on the profile ARNs) — the documented prerequisite for invoking through a profile (, read 2026-11-07). For every runtime Responses / Chat Completions target — any proprietary GPT CRIS id by id form (GPT-5.6 Sol / Terra / Luna, the GPT-6 family, GPT-5.5 / GPT-5.4: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`, or their `inference-profile/` ARNs) when `--runtime-api` is `responses` or `chat_completions` (or omitted) — it adds a third resource, `arn:aws:bedrock:::project/default`, because a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, read 2026-11-07) and each of those model cards (read 2026-11-07) repeats the requirement; do not strip it from the report's policy excerpt. A `converse` surface and every Anthropic / Nova / gpt-oss id get no project resource. The output is a ready-to-use IAM policy JSON file. + +**Runtime OpenAI and Runtime Messages lanes need no Mantle statement.** A `Rewrite strategy: runtime_openai` or `Rewrite strategy: runtime_messages` run keeps the source SDK but authorizes through `bedrock:InvokeModel*` (+ `GetInferenceProfile`, and the default-project ARN for every runtime Responses / Chat Completions GPT CRIS id above) on the `bedrock-runtime` endpoint; report exactly the statements the script produced and do not add `bedrock-mantle:*` for them. Per-token model pricing is the same on both endpoints (, read 2026-11-07), so the endpoint choice does not change the §6.3 cost table. **Mantle targets get different actions.** When any target is a bare proprietary OpenAI GPT model (GPT-5.x or `openai.gpt-6-astra`), the script also emits `bedrock-mantle:CreateInference` / `Get*` / `List*` scoped to `arn:aws:bedrock-mantle:::project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` (AWS does not permit narrowing that one). `bedrock:InvokeModel` alone does **not** authorize these models. Two consequences to carry into the report: diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md index 3f6f17a68..bdd9e8d99 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -88,15 +88,21 @@ diffs outputs, so the user validates behavior before cutover. ## Step 3: `setup_bedrock.sh` -Enables the chosen Bedrock model(s), sets up IAM, and (mantle path) the Bedrock API key. For a -`runtime_openai_cris` target whose CRIS id is a GPT-6 model (Astra, Sol, Luna, 6.1 Sol, 6.1 Terra) or -GPT-5.5 / GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / -`global.openai.gpt-5.4`, or their `inference-profile/` ARNs), the IAM -policy must grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` -in addition to the CRIS inference-profile ARN — both required per the GPT-6 model cards and the -GPT-5.5 / GPT-5.4 cards (read 2026-11-07); `iam_policy.py` emits the third ARN for exactly these -ids. The mantle `bedrock-mantle:*` policy (bare `openai.gpt-5.5` / `openai.gpt-5.4` ids included) -is unaffected. +Enables the chosen Bedrock model(s), sets up IAM, and (mantle path) the Bedrock API key. For +EVERY `runtime_openai_cris` target whose `required_client_api` is `responses` or +`chat_completions` — by id form, not model family: any proprietary GPT CRIS id, so GPT-5.6 Sol / +Terra / Luna, the GPT-6 family (Astra, Sol, Luna, 6.1 Sol, 6.1 Terra) and GPT-5.5 / GPT-5.4 +(`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or +their `inference-profile/` ARNs — the IAM policy must grant `bedrock:InvokeModel` on +`arn:aws:bedrock:{region}:{account}:project/default` in addition to the CRIS inference-profile +ARN: a Responses request on `bedrock-runtime` authorizes the default project separately from the +inference target (, +"Permissions", read 2026-11-07), and each of those model cards (read 2026-11-07) repeats the +requirement for its runtime CRIS ids. `iam_policy.py --runtime-api ` emits +the third ARN for exactly those rows and omits it for a `converse` row (the Converse-only +application-inference-profile route) and for every Anthropic / Nova / gpt-oss id. The mantle +`bedrock-mantle:*` policy (bare `openai.gpt-5.5` / `openai.gpt-5.4` / `openai.gpt-5.6-*` / +`openai.gpt-6-*` ids included) is unaffected. ## Step 3B: `gateway_config.{yaml,py,json}` (gateway users) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md index 0f35e8a4a..69db2ed08 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/openai-on-bedrock.md @@ -225,9 +225,13 @@ that expires within 12 hours and is not refreshed. Do not recommend it for produ **IAM:** on the mantle path, the managed policy `AmazonBedrockMantleInferenceAccess` grants what inference needs, including `bedrock-mantle:CreateInference` and `bedrock-mantle:CallWithBearerToken` — `bedrock:InvokeModel` does not authorize mantle calls. On the GPT-5.6 / GPT-5.5 / GPT-5.4 `bedrock-runtime` path the usual `bedrock:InvokeModel*` -against the CRIS inference-profile ARN applies, as for any other runtime model; the GPT-5.5 / GPT-5.4 cards add -that the identity also needs `bedrock:InvokeModel` on the account's default project -(`arn:aws:bedrock:{region}:{account-id}:project/default`). +against the CRIS inference-profile ARN applies, as for any other runtime model, and every runtime Responses / +Chat Completions target additionally needs `bedrock:InvokeModel` on the account's default project +(`arn:aws:bedrock:{region}:{account-id}:project/default`): the Responses API authorizes the default project +separately from the inference target +(, read 2026-11-07), and the +GPT-5.6 Sol / Terra / Luna cards as well as the GPT-5.5 / GPT-5.4 cards (read 2026-11-07) repeat it for their +runtime CRIS ids. A Converse-only route needs no project grant. **Reasoning effort:** the five GPT-5.x models accept `none`, `low`, `medium`, `high`, `xhigh`, `max`. Because these models reason before responding, the model's output items (which may include reasoning items) must be passed back in the next diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md index ab068ebac..178fc53cf 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md @@ -242,7 +242,9 @@ rubric file (`ai.md`, `ai-azure-openai-to-bedrock.md`, or a `vendored/ai/*` ref) records `required_client_api: "converse"` — not absent, `responses` or `chat_completions`. A same-workload Responses-only need (GPT-5.5 / GPT-5.4 structured outputs or implicit caching) is recorded as an explicit decision, not silently dropped. A region-only CRIS - move keeps the `responses` default. + move with no such need keeps the source app's own API as the default `required_client_api` + (`responses` for a Responses source, `chat_completions` for a Chat Completions source) — + never an implicit Converse. - [ ] If `ai_source` is `azure_openai` or `openai`: every source model that is available on Bedrock and carried by the target region maps to ITSELF with `model_change: false` — not to a Claude/Nova substitute — and records `migration_path: "runtime_openai_cris"` by default. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index e7ce5f31e..f9c7b04a4 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -334,9 +334,15 @@ limited to: `aws-bedrock-token-generator`) or a Bedrock API key, **not** an OpenAI key 3. Model ID → the CRIS inference-profile id (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), never the bare `openai.gpt-*` form -4. IAM → `bedrock:InvokeModel*` on the inference-profile and foundation-model ARNs (GPT-6 / GPT-5.5 / GPT-5.4 - also need `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account-id}:project/default`), plus - `bedrock:GetInferenceProfile` on the profile +4. IAM → `bedrock:InvokeModel*` on the inference-profile and foundation-model ARNs, plus + `bedrock:GetInferenceProfile` on the profile, plus — for EVERY runtime Responses / Chat Completions target, + i.e. every proprietary GPT CRIS id above (GPT-5.6 Sol / Terra / Luna, GPT-6, GPT-5.5, GPT-5.4) or its + `inference-profile/` ARN — `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account-id}:project/default`: + a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target + (, "Permissions", read + 2026-11-07), and each model card repeats the requirement for its runtime CRIS ids. + `iam_policy.py --runtime-api ` emits it for `responses` / `chat_completions` rows and + omits it for a `converse` row and for every non-OpenAI id **Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index 4a2e6fe9f..953869296 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -555,7 +555,9 @@ Write `aws-design-ai.json` to `$MIGRATION_DIR/`. records `required_client_api: "converse"` — not absent, `responses` or `chat_completions`. A same-workload Responses-only need (GPT-5.5 / GPT-5.4 structured outputs or implicit caching) is recorded as an explicit decision in the Part 5 plan, not silently dropped. - A region-only CRIS move keeps the `responses` default. + A region-only CRIS move with no such need keeps the source app's own API as the default + `required_client_api` (`responses` for a Responses source, `chat_completions` for a Chat + Completions source) — never an implicit Converse. - [ ] `regional_warnings` is present (empty array `[]` if no issues; populated if any service unavailable in target region) - [ ] `multi_model_warnings` is present (empty array `[]` if single model or no coordination issues) - [ ] If `agentic_profile.is_agentic == true`: `agentic_design` object is present with `migration_approach` matching `preferences.json` diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md index 137f8b153..08832c532 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md @@ -30,7 +30,7 @@ Check `preferences.json` → `ai_constraints.ai_framework` to determine the migr **OpenAI SDK users keeping the same model (config path for a Responses source; small reshape for a Chat Completions source)** — `ai_framework` = `direct` AND `ai_source` = `openai` AND (`migration_path` == `"runtime_openai_cris"` (the default) OR `migration_path` **starts with** `mantle` (`mantle`, `mantle_openai_responses`, `mantle_openai_chat`, selector-gated)): -- `runtime_openai_cris` (the default — GPT-5.6/GPT-6/GPT-5.5/GPT-5.4 via `bedrock-runtime` CRIS): point the client at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` — **not** the mantle URL. Update the model string to the CRIS inference-profile ID the selected model's card lists (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), not the bare ID. Grant standard `bedrock:InvokeModel*` actions against the CRIS inference-profile ARN — **not** `bedrock-mantle:*`. For a GPT-6 CRIS id, and for the GPT-5.5 / GPT-5.4 CRIS ids (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`, or their `inference-profile/` ARNs), also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` alongside the inference-profile ARN (per the GPT-6 model cards and the GPT-5.5 / GPT-5.4 cards, read 2026-11-07); `iam_policy.py` emits this third ARN for exactly those ids. Read `aws-design-ai.json` → `ai_architecture.bedrock_models[].required_client_api` — the API the source app already calls — and emit that surface (`responses.create`, `chat.completions.create`, or Converse when Design recorded `converse` for a Guardrails or application-inference-profile need); for a region-only move it is simply the source's own API. +- `runtime_openai_cris` (the default — GPT-5.6/GPT-6/GPT-5.5/GPT-5.4 via `bedrock-runtime` CRIS): point the client at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` — **not** the mantle URL. Update the model string to the CRIS inference-profile ID the selected model's card lists (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), not the bare ID. Grant standard `bedrock:InvokeModel*` actions against the CRIS inference-profile ARN — **not** `bedrock-mantle:*`. For every runtime Responses / Chat Completions target — any proprietary GPT CRIS id by id form: GPT-5.6 Sol / Terra / Luna, the GPT-6 family, and GPT-5.5 / GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` alongside the inference-profile ARN: the Responses API authorizes the default project separately from the inference target (, read 2026-11-07) and each model card repeats it for its runtime CRIS ids; `iam_policy.py --runtime-api responses|chat_completions` emits this third ARN for exactly those ids, and omits it for a `converse` row. Read `aws-design-ai.json` → `ai_architecture.bedrock_models[].required_client_api` — the API the source app already calls — and emit that surface (`responses.create`, `chat.completions.create`, or Converse when Design recorded `converse` for a Guardrails or application-inference-profile need); for a region-only move it is simply the source's own API. - `mantle` / `mantle_openai_responses` / `mantle_openai_chat` (only when Design recorded a `mantle_selector`, see `references/vendored/ai/bedrock-endpoint-policy.md`): point the client at `https://bedrock-mantle.{region}.api.aws/openai/v1` — note the `openai/v1` segment for proprietary GPT models; a bare `/v1` 404s. Swap the credential for a Bedrock API key or the auto-refreshing token provider — **not** the existing OpenAI key. Update the model string to the bare Bedrock model ID (`openai.gpt-*` for a same-model move). Grant the `bedrock-mantle:*` actions; `bedrock:InvokeModel` does not authorize these models. - **Config-only if the source already calls `responses.create`.** A Chat Completions source for GPT-6 or GPT-5.6 may keep `chat.completions.create`/`choices[0].message` parsing unchanged — Chat Completions is supported on both mantle and runtime CRIS for these models, so reshape to Responses is optional (only for a Responses-only feature or demonstrated constraint). GPT-5.5/GPT-5.4 follow the same rule — their model cards (read 2026-11-06) list Chat Completions on both mantle and runtime, so the reshape is optional for them too; probe the selected API against the target account before committing, as for any model. - Test in staging, validate responses diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md index ab7569d29..905cfb285 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -57,10 +57,10 @@ Check `preferences.json` → `ai_constraints.ai_framework.value` and `aws-design ID, the `bedrock-runtime` base URL `https://bedrock-runtime.{region}.amazonaws.com/openai/v1`, `bedrock:InvokeModel*` IAM and the row's `required_client_api`) + setup (Step 3) + test harness (Step 2). Skip provider adapter (Step 1). -- `migration_path` starts with `"mantle"` (`"mantle"`, `"mantle_openai_chat"`, `"mantle_openai_responses"`) → Generate Mantle setup (Step 1M) + setup (Step 3) + test harness (Step 2). Skip provider adapter (Step 1). Design records `"mantle_openai_responses"` or `"mantle_openai_chat"` only together with `bedrock_models[].mantle_selector` (see `references/vendored/ai/bedrock-endpoint-policy.md`); echo that selector in the generated script's header so the Mantle choice stays traceable. Treat the `"mantle_openai_responses"` spelling as this same Mantle path. +- `migration_path` starts with `"mantle"` AND is one of the three OpenAI-SDK spellings — exactly `"mantle"`, `"mantle_openai_chat"` or `"mantle_openai_responses"` (`"mantle_messages"` is NOT this branch; it has its own entry below) → Generate Mantle setup (Step 1M) + setup (Step 3) + test harness (Step 2). Skip provider adapter (Step 1). Design records `"mantle_openai_responses"` or `"mantle_openai_chat"` only together with `bedrock_models[].mantle_selector` (see `references/vendored/ai/bedrock-endpoint-policy.md`); echo that selector in the generated script's header so the Mantle choice stays traceable. Treat the `"mantle_openai_responses"` spelling as this same Mantle path. - `migration_path` = `"gpt-oss"` → Generate provider adapter (Step 1) targeting the gpt-oss model on Bedrock via the Converse API + setup (Step 3) + test harness (Step 2). Use the Bedrock model ID from `aws-design-ai.json` → `ai_architecture.bedrock_models[].aws_model_id` (the gpt-oss model ID). Do NOT generate a Mantle script — gpt-oss uses the Converse API directly. - `migration_path` = `"runtime_messages"` (Anthropic source; Design's default for an Anthropic SDK app) → Generate provider adapter (Step 1) whose Bedrock provider class keeps the Anthropic SDK at the runtime base URL `https://bedrock-runtime.{region}.amazonaws.com/anthropic` with a string Bedrock token (`provide_token(region=...)` minted per client) and the row's `global.`/Geo Claude profile ID + setup (Step 3, `bedrock:InvokeModel*` and `bedrock:GetInferenceProfile` IAM) + test harness (Step 2). `required_client_api` is `"messages"`; do NOT generate a Mantle script and do NOT rewrite to Converse. -- `migration_path` = `"mantle_messages"` (Anthropic source, only with `bedrock_models[].mantle_selector`, normally `workspaces`) → the same adapter shape at the Mantle base URL `https://bedrock-mantle.{region}.api.aws/anthropic` with the bare `anthropic.claude-*` ID and `bedrock-mantle:*` IAM; echo the selector in the adapter header. +- `migration_path` = `"mantle_messages"` (Anthropic source, only with `bedrock_models[].mantle_selector`, normally `workspaces`; matched by this entry ONLY, never by the OpenAI-SDK Mantle branch above) → Generate provider adapter (Step 1) with the same Anthropic-SDK adapter shape as `runtime_messages` but at the Mantle base URL `https://bedrock-mantle.{region}.api.aws/anthropic` with the bare `anthropic.claude-*` ID and `bedrock-mantle:*` IAM (Step 3); `required_client_api` stays `"messages"`; echo the selector in the adapter header. Do NOT generate `migrate_to_mantle.sh` (Step 1M) for it. - `"converse"`, `"direct"`, or absent → Generate provider adapter (Step 1) + setup (Step 3) + test harness (Step 2). `"converse"` is an explicit boto3 Converse rewrite (Design records it on request or for a Converse-only feature); do NOT generate a Mantle script for it. - `"llm_router"`, `"api_gateway"`, `"voice_platform"`, or `"framework"` → Skip Step 1, generate gateway config (Step 3B) instead. Step 3B's OpenRouter subroute can still replace that gateway config with Step 1M (`same_model_runtime` → the CRIS variant, `same_model_mantle` → the Mantle variant) or the provider adapter (`direct`); do not decide those from `migration_path` alone. @@ -215,7 +215,7 @@ Generate `ai-migration/setup_bedrock.sh`. - Dry-run by default (`--execute` flag to run for real) - Step 1 — Request model access: List each model from `aws-design-ai.json` → `bedrock_models[].aws_model_id` and the embedding model -- Step 2 — Create IAM role: Trust policy for the compute platform (Lambda, ECS, or EC2 based on `aws-design.json` if present). Match inference permissions to the selected endpoint. Mantle needs `bedrock-mantle:CreateInference` scoped to `arn:aws:bedrock-mantle:{region}:{account_id}:project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` when using bearer credentials. Runtime needs `bedrock:InvokeModel` / `bedrock:InvokeModelWithResponseStream` against the selected foundation-model ARN and, for CRIS, the exact inference-profile ARN. For GPT-6 CRIS, and for the GPT-5.5 / GPT-5.4 CRIS ids (`us.`/`global.openai.gpt-5.5`, `us.`/`global.openai.gpt-5.4`, or their `inference-profile/` ARNs — their cards, read 2026-11-07, require it "in addition to the inference profile"), also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default`. A bare `openai.gpt-5.5` / `openai.gpt-5.4` id is a mantle target and gets no such grant. Runtime actions do not authorize Mantle. +- Step 2 — Create IAM role: Trust policy for the compute platform (Lambda, ECS, or EC2 based on `aws-design.json` if present). Match inference permissions to the selected endpoint. Mantle needs `bedrock-mantle:CreateInference` scoped to `arn:aws:bedrock-mantle:{region}:{account_id}:project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` when using bearer credentials. Runtime needs `bedrock:InvokeModel` / `bedrock:InvokeModelWithResponseStream` against the selected foundation-model ARN and, for CRIS, the exact inference-profile ARN. For EVERY `runtime_openai_cris` row whose `required_client_api` is `responses` or `chat_completions` — classified by id form, not model family: any `us.`/`global.` (and, for GPT-5.6 Terra and Luna, `in.`) `openai.gpt-*` CRIS id, so GPT-5.6 Sol / Terra / Luna, the GPT-6 family, GPT-5.5 and GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default`: a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, "Permissions", read 2026-11-07), and each of those model cards (read 2026-11-07) repeats the requirement for its runtime CRIS ids. `iam_policy.py --runtime-api ` emits that third ARN for exactly those rows. A row with `required_client_api: "converse"` (the Converse-only application-inference-profile route) and every Anthropic / Nova / gpt-oss id get no such grant; a bare `openai.gpt-5.5` / `openai.gpt-5.4` / `openai.gpt-5.6-*` / `openai.gpt-6-*` id is a mantle target and gets no such grant. Runtime actions do not authorize Mantle. - If the runtime client uses `BEDROCK_INFERENCE_PROFILE_ARN` from the generated monitoring output, also authorize that exact `application-inference-profile/…` ARN, retaining the backing model/CRIS resources needed for the selected model and fallback. Resolve it from the same output used by the client; do not grant every application profile. Re-run setup after enabling or changing this override so the effective invocation target and IAM remain aligned. See [inference-profile IAM prerequisites](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-prereq.html). - Compliance carry-through: if `preferences.json` → `design_constraints.compliance` is set and not `none`/`unknown` (full-flow Q2 / AI-only Q1.5), the script and its printed notes MUST reflect the design's Step 0.7 constraints — `AWS_REGION` from the compliance-constrained region (GovCloud for fedramp, EU for gdpr), `eu.` inference-profile model IDs for gdpr, and for hipaa a printed warning that Bedrock invocation logs retain original content (KMS-encrypt + restrict IAM on the log group before enabling) - Step 3 — Print required environment variables: `AWS_REGION`, `AI_PROVIDER=bedrock`, model IDs diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md index d986ee273..58dc0635e 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/openai-on-bedrock.md @@ -229,9 +229,13 @@ that expires within 12 hours and is not refreshed. Do not recommend it for produ **IAM:** on the mantle path, the managed policy `AmazonBedrockMantleInferenceAccess` grants what inference needs, including `bedrock-mantle:CreateInference` and `bedrock-mantle:CallWithBearerToken` — `bedrock:InvokeModel` does not authorize mantle calls. On the GPT-5.6 / GPT-5.5 / GPT-5.4 `bedrock-runtime` path the usual `bedrock:InvokeModel*` -against the CRIS inference-profile ARN applies, as for any other runtime model; the GPT-5.5 / GPT-5.4 cards add -that the identity also needs `bedrock:InvokeModel` on the account's default project -(`arn:aws:bedrock:{region}:{account-id}:project/default`). +against the CRIS inference-profile ARN applies, as for any other runtime model, and every runtime Responses / +Chat Completions target additionally needs `bedrock:InvokeModel` on the account's default project +(`arn:aws:bedrock:{region}:{account-id}:project/default`): the Responses API authorizes the default project +separately from the inference target +(, read 2026-11-07), and the +GPT-5.6 Sol / Terra / Luna cards as well as the GPT-5.5 / GPT-5.4 cards (read 2026-11-07) repeat it for their +runtime CRIS ids. A Converse-only route needs no project grant. **Reasoning effort:** the five GPT-5.x models accept `none`, `low`, `medium`, `high`, `xhigh`, `max`. Astra's card does not enumerate these values; verify its accepted settings rather than inheriting the GPT-5.x list. Because these models reason diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index e7ce5f31e..f9c7b04a4 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -334,9 +334,15 @@ limited to: `aws-bedrock-token-generator`) or a Bedrock API key, **not** an OpenAI key 3. Model ID → the CRIS inference-profile id (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), never the bare `openai.gpt-*` form -4. IAM → `bedrock:InvokeModel*` on the inference-profile and foundation-model ARNs (GPT-6 / GPT-5.5 / GPT-5.4 - also need `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account-id}:project/default`), plus - `bedrock:GetInferenceProfile` on the profile +4. IAM → `bedrock:InvokeModel*` on the inference-profile and foundation-model ARNs, plus + `bedrock:GetInferenceProfile` on the profile, plus — for EVERY runtime Responses / Chat Completions target, + i.e. every proprietary GPT CRIS id above (GPT-5.6 Sol / Terra / Luna, GPT-6, GPT-5.5, GPT-5.4) or its + `inference-profile/` ARN — `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account-id}:project/default`: + a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target + (, "Permissions", read + 2026-11-07), and each model card repeats the requirement for its runtime CRIS ids. + `iam_policy.py --runtime-api ` emits it for `responses` / `chat_completions` rows and + omits it for a `converse` row and for every non-OpenAI id **Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md b/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md index e2897cd7b..87d5bb904 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md @@ -709,11 +709,30 @@ work to the telemetry hooks. Read `$MIGRATION_DIR/aws-design-ai.json` and extract: -- `ai_architecture.bedrock_models[]` → array of `{source_model, aws_model_id, use_case}` -- Collect all `aws_model_id` values into `$TARGET_MODELS` (array). Keep the `use_case` of each: - the preflight script probes each model by the right API automatically (Converse for chat, - InvokeModel for embeddings), but the evaluator's quality scoring only applies to chat models — - embedding targets get format/dimension validation only. +- `ai_architecture.bedrock_models[]` → array of `{source_model, aws_model_id, use_case, + migration_path, required_client_api, mantle_selector}` (the last three may be absent on older plans) +- Collect all `aws_model_id` values into `$TARGET_MODELS` (array). Keep the `use_case` AND the + `required_client_api` of each row: B4 passes every row to preflight as its own + `=` pair, so a Claude Messages primary and a GPT Chat or Nova + Converse secondary are each probed on their own API (embedding rows are always probed via + InvokeModel; pass `invoke` for them). The evaluator's quality scoring only applies to chat + models — embedding targets get format/dimension validation only. +- **Lane gate (before B2, B4 and every Phase C dispatch).** Run + + ```bash + uv run --project $SCRIPTS python $SCRIPTS/preflight_bedrock.py --check-lanes $MIGRATION_DIR/aws-design-ai.json + ``` + + Exit 1 with `reason: unsupported_lane` means a row's `migration_path` is `mantle_messages` — + the deferred **Mantle Messages** execution lane (Anthropic SDK on + `https://bedrock-mantle.{region}.api.aws/anthropic`), which Design records only for the + `workspaces` mantle selector. This skill has no preflight, evaluation or rewrite route for that + lane: STOP here. Show the user the result's `lane`, `mantle_selector` and `failing_models`, + say the lane is not yet supported by llm-to-bedrock, and offer to re-run Design without the + Workspaces requirement so the row takes the supported `runtime_messages` route. Do NOT run B4 + against the bare Claude id, do NOT dispatch C1–C6, and do NOT generate IAM for it + (`iam_policy.py --migration-path mantle_messages` refuses with the same `unsupported_lane` + reason). Exit 0 (`lanes_supported`) → continue. Read `$MIGRATION_DIR/ai-workload-profile.json` and extract: @@ -830,19 +849,36 @@ reliably visible, and even that must be materialized to the file for the baselin ### B4 — Bedrock preflight +**Resolve the primary row's API first (`$RESOLVED_RUNTIME_API`).** The primary chat row's +`required_client_api` (older plans: that row's `runtime_api`, then +`ai_architecture.code_migration.runtime_api`) is the API B4, the context block and C0 all use. +When the plan records one, `$RESOLVED_RUNTIME_API` is that value. When a `runtime_openai_cris` +plan records none: on resume, read `resolved_runtime_api` from the saved +`$REPO/.saws-migrate/phase-results/run-context.json` (C0's `$PHASE_DIR`) and reuse it without +asking; otherwise AskUserQuestion "Which API does the source app call: +Responses or Chat Completions?" and set `$RESOLVED_RUNTIME_API` to the answer. Other paths that +record no API resolve from the path itself: `runtime_messages` → `messages`; +`mantle_openai_chat` → `chat_completions`; `mantle` / `mantle_openai_responses` → `responses` +(a bare GPT id is probed on the Mantle Responses API either way); `converse` / `direct` / +`gpt-oss` → `converse`. `$RESOLVED_RUNTIME_API` is therefore always one of `responses`, +`chat_completions`, `converse`, `messages`. Never fall back to Converse by omission — a +`runtime_openai_cris` row without an API is resolved here, not guessed. + ```bash -uv run --project $SCRIPTS python $SCRIPTS/preflight_bedrock.py --region $REGION --models --dataset-size 200 --runtime-api +uv run --project $SCRIPTS python $SCRIPTS/preflight_bedrock.py --region $REGION --target = [--target ... one per bedrock_models[] row] --dataset-size 200 ``` (`--dataset-size 200` matches the golden-dataset cap, so the quota warning reflects the worst case. Prefix with `AWS_PROFILE=$AWS_PROFILE_CHOICE` if B2 chose a non-default profile.) -`--runtime-api` is the selected row's `required_client_api`: pass `messages` when it is -`"messages"` (a `runtime_messages` plan — the probe uses the Anthropic SDK against the runtime -`/anthropic` endpoint), `responses` or `chat_completions` for the runtime OpenAI-compatible -APIs, and `converse` only when the plan recorded `"converse"`. For a `runtime_openai_cris` plan -ALWAYS pass the explicit source API (`responses` | `chat_completions`, or `converse` only when -recorded) — never omit the flag to fall back to the Converse default; a plan with no recorded -API is resolved first by the context-block rule below, not guessed here. +Each `--target` pair carries ITS OWN row's API, so a mixed plan is probed correctly: the +primary row gets `$RESOLVED_RUNTIME_API`; a secondary row gets its own `required_client_api` +(`messages` → Anthropic SDK against the runtime `/anthropic` endpoint; `responses` / +`chat_completions` → the runtime OpenAI-compatible APIs; `converse` → Converse); an embedding +row gets `invoke`. A row whose API is missing or unknown makes preflight fail closed +(`reason: unknown_runtime_api`, nothing probed) — resolve that row's API in the saved plan, +do not substitute Converse. The legacy single-API form +`--models --runtime-api $RESOLVED_RUNTIME_API` is still +accepted for a plan with exactly one chat row and no secondary targets. Parse the JSON output. On failure the TOP LEVEL carries `reason`/`detail` (lifted from the first failing model) plus `failing_models` (all failing ids); per-model verdicts are in `models[]`: @@ -854,6 +890,7 @@ first failing model) plus `failing_models` (all failing ids); per-model verdicts - `ok == false` + `reason: openai_deps_missing` → the selected runtime Chat/Responses API could not be probed because its SDK is missing. Re-sync the same scripts environment and stop; do not substitute Converse. - `ok == false` + `reason: messages_deps_missing` → the selected runtime Anthropic Messages API could not be probed because `anthropic` / `aws-bedrock-token-generator` is missing from the pinned scripts environment. Re-sync (`uv sync --project $SCRIPTS`) and stop; do not substitute Converse — access was never verified. - `ok == false` + `reason: model_unavailable` → Read the `resolve-bedrock-model-id` reference at `$HELPERS/resolve-bedrock-model-id/resolve-bedrock-model-id.md` and follow its procedure with each ID from `failing_models` + region. AskUserQuestion with the candidates: "Use `` (cross-region inference profile)" / "Paste a different model ID" / "Abort". On a choice, replace the ID in `$TARGET_MODELS` and re-run B4. +- `ok == false` + `reason: unknown_runtime_api` → a `--target` row's API was missing or not one of `converse` / `responses` / `chat_completions` / `messages` (`invoke` for embeddings). Nothing was probed. Resolve that row's `required_client_api` (the B4 resolution step above for the primary row; the saved plan for a secondary row) and re-run B4; never guess Converse. - `ok == false` + any other `reason` → show `detail` and stop. - `ok == true` → proceed. Surface any `quota_warning`, and any model whose `reason` is `embedding_unprobed` (embedding family the preflight can't probe — remind the user to confirm @@ -902,20 +939,24 @@ resolved_api_path = normalize_api_path( plan_migration_path, validated_target_model_ids, plan_runtime_api) ``` -Read `plan_runtime_api` from the selected model row's `required_client_api`. For older -plans, fall back to that row's `runtime_api`, then `ai_architecture.code_migration.runtime_api`. -An explicit `invoke` surface is unsupported by this execution lane: stop for an explicit -supported API decision rather than treating it as Converse. A `runtime_openai_cris` plan -resolves to `runtime_openai_chat` or `runtime_openai_responses` from that explicit API and -**fails closed when none is recorded**: `normalize_api_path` raises `ValueError` ("Converse is -never the implicit default"). When that happens, do not catch it and guess — AskUserQuestion -"Which API does the source app call: Responses or Chat Completions?", record the answer in -run-context as `resolved_runtime_api`, use it as `plan_runtime_api` and re-normalize (on -resume, read `resolved_runtime_api` from run-context before asking again). An explicit +`plan_runtime_api` is B4's `$RESOLVED_RUNTIME_API`: the selected +model row's `required_client_api`, for older plans that row's `runtime_api`, then +`ai_architecture.code_migration.runtime_api`, and for a `runtime_openai_cris` plan that records +none, the user's B4 answer. An explicit `invoke` surface is unsupported by this execution lane: +stop for an explicit supported API decision rather than treating it as Converse. A +`runtime_openai_cris` plan resolves to `runtime_openai_chat` or `runtime_openai_responses` from +that explicit API and **fails closed when none is recorded**: `normalize_api_path` raises +`ValueError` ("Converse is never the implicit default"). Reaching that error here means B4's +resolution step was skipped — do not catch it and guess; go back to B4, resolve +`$RESOLVED_RUNTIME_API` (on resume, read `resolved_runtime_api` from run-context before asking +again), and re-normalize. The same value is written to current-context.json as +`resolved_runtime_api` (C0), so an unchanged resume matches without re-asking. An explicit `required_client_api: "converse"` is the ONLY route to a Converse rewrite for a `runtime_openai_cris` plan. `runtime_messages` (Anthropic SDK on the runtime `/anthropic` -endpoint) and `mantle_messages` pass through `normalize_api_path` unchanged. Do not ignore -this field on dispatch or resume. +endpoint) passes through `normalize_api_path` unchanged. `mantle_messages` never reaches this +block: the B1 lane gate (`preflight_bedrock.py --check-lanes`) halts it as an unsupported lane +before B4 and before any dispatch — it is NOT a Converse plan and must not be labelled one. +Do not ignore this field on dispatch or resume. The legacy `migration_path: "mantle"` alias with bare proprietary GPT targets resolves to `mantle_openai_responses`, so evaluator and rewriter keep the documented Responses default. Other explicit `migration_path` values are preserved. With no path, an all-bare proprietary GPT target @@ -933,7 +974,7 @@ AWS region: <$REGION> AWS profile (pass as --profile / AWS_PROFILE= inline on every aws/boto3 invocation): <$AWS_PROFILE_CHOICE — omit line if default> Target Bedrock model(s): Target API path: -Target API surface: +Target API surface: Migration plan dir: <$MIGRATION_DIR> Resolved target model id: Scripts directory (pinned uv toolchain): <$SCRIPTS> @@ -986,6 +1027,7 @@ Bedrock at their expense, capped at 200 cases. "repo_dirty_sha256": "", "target_models": [{"source_model": "...", "aws_model_id": "...", "use_case": "..."}], "resolved_model_overrides": {}, + "resolved_runtime_api": "<$RESOLVED_RUNTIME_API from B4: responses | chat_completions | converse | messages — ALWAYS present, so saved and current contexts compare field-for-field>", "source_provider": "", "source_baseline_available": , "source_key_sha256": "", @@ -1039,6 +1081,7 @@ uv run --project $SCRIPTS python $SCRIPTS/validate_result.py --check-run-context | repo_root, migration_dir, region, aws_profile, aws_account, source_provider, assess_design_sha256, schema_version, plugin_version | everything | — | | repo_head_sha / repo_branch / repo_dirty_sha256 | everything | — | | target_models / resolved_model_overrides | ANALYSIS, EVAL, REWRITE, REPORT | INGESTION | +| resolved_runtime_api | EVAL, REWRITE, REPORT | ANALYSIS, INGESTION | | evaluation_contract_version | ANALYSIS, EVAL, REWRITE, REPORT | INGESTION | | log_files / max_golden_cases | everything | — | | source_key_sha256 / source_baseline_available | ANALYSIS, EVAL, REWRITE, REPORT | INGESTION | diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py index 9e194569e..63624ff6d 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py @@ -8,7 +8,19 @@ import re import sys -_GEO_PREFIX = re.compile(r"^(us|eu|apac|au|jp|global)\.") +_GEO_PREFIX = re.compile(r"^(us|eu|apac|au|jp|in|us-gov|global)\.") + +# The runtime APIs a plan can select (bedrock_models[].required_client_api). The +# OpenAI-compatible pair is what the default-project grant keys on. +RUNTIME_APIS = frozenset({"converse", "responses", "chat_completions", "messages"}) +_OPENAI_COMPATIBLE_APIS = frozenset({"responses", "chat_completions"}) + +# Execution lanes this helper refuses to authorize. `mantle_messages` (Anthropic SDK +# on the bedrock-mantle `/anthropic` endpoint, selected by the `workspaces` mantle +# selector) is deferred: llm-to-bedrock has no preflight, evaluation or rewrite route +# for it, so emitting runtime `bedrock:InvokeModel*` for its bare Claude id would +# authorize the wrong endpoint. Callers stop before dispatch instead. +UNSUPPORTED_LANES = frozenset({"mantle_messages"}) def is_inference_profile(model_id: str) -> bool: @@ -61,10 +73,13 @@ def inference_profile_arn(model_id: str, region: str, account_id: str) -> str: def default_project_arn(region: str, account_id: str) -> str: """ARN for the default Bedrock project, scoped to the account/region. - Required alongside the inference-profile ARN for a GPT-6 CRIS id and for the - GPT-5.5 / GPT-5.4 CRIS ids on bedrock-runtime: every GPT-6 model card's "Call - the model" section, and the GPT-5.4 / GPT-5.5 cards (read 2026-11-07), grant - `bedrock:InvokeModel` on both the inference-profile ARN and this ARN.""" + A Responses request on bedrock-runtime authorizes `bedrock:InvokeModel` on the + inference target AND on this ARN + (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, + "Permissions", read 2026-11-07). Every proprietary GPT model card — GPT-5.6 Sol / + Terra / Luna, GPT-6, GPT-5.5, GPT-5.4 (read 2026-11-07) — repeats it for its + runtime CRIS ids: the identity "also needs bedrock:InvokeModel on your account's + default project ... in addition to the inference profile".""" return f"arn:{_partition(region)}:bedrock:{region}:{account_id}:project/default" @@ -94,22 +109,65 @@ def is_gpt6_cris_model(model_id: str) -> bool: def is_gpt54_55_cris_model(model_id: str) -> bool: """True for exactly `us.`/`global.openai.gpt-5.4` / `-5.5`, as a short id or as an `arn:...:inference-profile/` ARN. The bare `openai.gpt-5.4` / `-5.5` ids - (mantle) and every other model, GPT-5.6 CRIS ids included, do NOT match.""" + (mantle) and every other model, GPT-5.6 CRIS ids included, do NOT match. This + is a model-family predicate (the exact ids those two cards publish); the + default-project grant itself is decided by `requires_default_project_grant`.""" if not is_inference_profile(model_id): return False return model_id.rsplit("/", 1)[-1].lower() in _GPT54_55_CRIS_IDS -def requires_default_project_grant(model_id: str) -> bool: - """True when the model card for this runtime CRIS id requires - `bedrock:InvokeModel` on the default project ARN alongside the - inference-profile ARN: every GPT-6 CRIS id, and the GPT-5.5 / GPT-5.4 ones.""" - return is_gpt6_cris_model(model_id) or is_gpt54_55_cris_model(model_id) +def is_runtime_openai_cris_model(model_id: str) -> bool: + """True for every proprietary GPT id in its bedrock-runtime CRIS form: a + `us.` / `in.` / `us-gov.` / `global.` prefixed `openai.gpt-*` id (GPT-5.6 Sol, + Terra and Luna, the GPT-6 family, GPT-5.5, GPT-5.4) or its + `arn:...:inference-profile/` ARN. Bare ids (mantle), the open-weight + gpt-oss models and every non-OpenAI id do NOT match — classification is by id + form, not by model family.""" + if not is_inference_profile(model_id): + return False + base_id = _GEO_PREFIX.sub("", model_id.rsplit("/", 1)[-1]).lower() + return base_id.startswith("openai.gpt-") and "oss" not in base_id + + +def requires_default_project_grant(model_id: str, runtime_api: str | None = None) -> bool: + """True when the policy must grant `bedrock:InvokeModel` on the default project + ARN alongside the inference-profile ARN for this target. + + The grant belongs to the runtime OpenAI-compatible path, not to a model + family: a Responses request on bedrock-runtime authorizes the default project + separately from the inference target (inference-responses-api.html, + "Permissions", read 2026-11-07), and every proprietary GPT card — GPT-5.6 Sol / + Terra / Luna, GPT-6, GPT-5.5, GPT-5.4 — attaches the same requirement to its + runtime CRIS ids for both the Responses and the Chat tab of its sample code. + Decided by the SELECTED API when the pipeline knows it: + - `responses` / `chat_completions` → grant for every runtime GPT CRIS target + - `converse` / `messages` → no grant (inference-prereq.html lists only + `bedrock:InvokeModel*` + `bedrock:GetInferenceProfile` for Converse through a + profile; Anthropic Messages targets are not OpenAI-compatible) + - None (id-only caller) → grant for every runtime GPT CRIS target, the + conservative reading of the cards when the path is unknown + Bare mantle ids, gpt-oss and every non-OpenAI id never get it.""" + if not is_runtime_openai_cris_model(model_id): + return False + if runtime_api is None: + return True + return runtime_api in _OPENAI_COMPATIBLE_APIS -def generate_policy(model_ids: list[str], region: str, account_id: str) -> dict: +def generate_policy(model_ids: list[str], region: str, account_id: str, + runtime_api: str | None = None, + migration_path: str | None = None) -> dict: """Build a scoped IAM policy covering exactly the given model IDs. + `runtime_api` is the plan's selected API (`responses`, `chat_completions`, + `converse`, `messages`); it decides the default-project grant (see + `requires_default_project_grant`). None keeps the id-only behaviour. + `migration_path` is the plan's `code_migration.migration_path`; an + UNSUPPORTED lane (`mantle_messages`) raises `ValueError` instead of emitting + runtime `bedrock:InvokeModel*` for a bare Claude id that the lane would never + send to bedrock-runtime. + Emits up to four statements, depending on which endpoints the targets use: - `bedrock:InvokeModel*` scoped to foundation-model ARNs (plain IDs) and inference-profile ARNs (geo-prefixed IDs), for bedrock-runtime targets @@ -126,6 +184,15 @@ def generate_policy(model_ids: list[str], region: str, account_id: str) -> dict: all-mantle run must not emit an InvokeModel statement with an empty Resource list, which is an invalid policy. """ + if migration_path in UNSUPPORTED_LANES: + raise ValueError( + f"migration_path {migration_path!r} is a deferred execution lane (Anthropic SDK on " + f"the bedrock-mantle /anthropic endpoint, selected by the `workspaces` mantle " + f"selector); llm-to-bedrock has no preflight, evaluation or rewrite route for it " + f"and this helper emits no runtime permissions for its bare Claude id") + if runtime_api is not None and runtime_api not in RUNTIME_APIS: + raise ValueError(f"Unknown runtime API {runtime_api!r}; expected one of " + f"{sorted(RUNTIME_APIS)}") runtime_ids = [m for m in model_ids if not is_mantle_model(m, region)] mantle_ids = [m for m in model_ids if is_mantle_model(m, region)] @@ -137,12 +204,13 @@ def generate_policy(model_ids: list[str], region: str, account_id: str) -> dict: resources.append(inference_profile_arn(mid, region, account_id)) base_id = _GEO_PREFIX.sub("", mid.rsplit("/", 1)[-1]) resources.append(foundation_model_arn(base_id, region)) - # A GPT-6 CRIS id's model card "Call the model" section, and the - # GPT-5.5 / GPT-5.4 cards for their us./global. CRIS ids, grant + # A runtime Responses / Chat Completions target (any proprietary GPT + # CRIS id — GPT-5.6, GPT-6, GPT-5.5, GPT-5.4 — by id form) needs # InvokeModel on the inference-profile ARN AND the default project - # ARN — both are required, neither replaces the other. GPT-5.6 CRIS - # ids and all other runtime targets keep the 2-ARN shape above. - if requires_default_project_grant(mid): + # ARN; both are required, neither replaces the other. Converse-only + # routes, Anthropic ids and every other runtime target keep the + # 2-ARN shape above. + if requires_default_project_grant(mid, runtime_api): resources.append(default_project_arn(region, account_id)) else: resources.append(foundation_model_arn(mid, region)) @@ -204,10 +272,26 @@ def generate_policy(model_ids: list[str], region: str, account_id: str) -> dict: parser.add_argument("--region", required=True, help="AWS region") parser.add_argument("--account-id", required=True, help="AWS account ID") parser.add_argument("--output", help="Output file (default: stdout)") + parser.add_argument("--runtime-api", choices=sorted(RUNTIME_APIS), + help="Selected runtime API (bedrock_models[].required_client_api); " + "responses/chat_completions add the default-project grant for " + "runtime GPT CRIS ids, converse/messages do not. Omit for the " + "id-only behaviour.") + parser.add_argument("--migration-path", + help="Plan's code_migration.migration_path; an unsupported lane " + "(mantle_messages) exits 1 with a JSON reason instead of a policy.") args = parser.parse_args() model_ids = [m.strip() for m in args.models.split(",") if m.strip()] - policy = generate_policy(model_ids, args.region, args.account_id) + try: + policy = generate_policy(model_ids, args.region, args.account_id, + runtime_api=args.runtime_api, + migration_path=args.migration_path) + except ValueError as e: + print(json.dumps({"ok": False, "reason": "unsupported_lane" + if args.migration_path in UNSUPPORTED_LANES else "unknown_runtime_api", + "lane": args.migration_path, "detail": str(e)}, indent=2)) + sys.exit(1) output = json.dumps(policy, indent=2) + "\n" if args.output: diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py index 1b4c31b79..c6effec95 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py @@ -1,15 +1,28 @@ # preflight_bedrock.py """Bedrock fail-fast preflight: authorization, region/model availability, quota. -Usage: python preflight_bedrock.py --region --models [--dataset-size N] +Usage: + python preflight_bedrock.py --region --target = [--target ...] [--dataset-size N] + python preflight_bedrock.py --region --models [--runtime-api ] [--dataset-size N] + python preflight_bedrock.py --check-lanes Prints a JSON verdict to stdout; exit 0 if all models pass, 1 otherwise. Always prints JSON — including for missing credentials / bad region — so the caller can parse the verdict instead of a traceback. On failure the top level carries `reason`/`detail`/`failing_models` lifted from the first failing model. -Each model is probed on the API the plan selected (--runtime-api): `converse` -(the default when no API was selected), `responses` / `chat_completions` for the -runtime OpenAI-compatible APIs, or `messages` for the runtime Anthropic Messages -API. Embedding models (which don't speak Converse) go via InvokeModel with their +Each model is probed on the API ITS OWN design row selected. `--target ID=API` +carries that per-row API (`bedrock_models[].required_client_api`): `converse`, +`responses` / `chat_completions` for the runtime OpenAI-compatible APIs, +`messages` for the runtime Anthropic Messages API, or `invoke` for an embedding +row. A row whose API is missing or unknown fails CLOSED (`reason: +unknown_runtime_api`) before any probe runs — a mixed plan (Claude Messages +primary + GPT Chat or Nova Converse secondary) must never send every target to +one endpoint. `--models` + `--runtime-api` is the legacy single-API form and +applies that one API to every listed id (`converse` when omitted). +`--check-lanes` reads the design's `bedrock_models[]` rows and stops with +`reason: unsupported_lane` when any row's `migration_path` is a deferred lane +(`mantle_messages`, the Workspaces selector's Anthropic-SDK-on-Mantle route); +run it before preflight or any Phase C dispatch. +Embedding models (which don't speak Converse) go via InvokeModel with their family's request body. Bare OpenAI proprietary GPT ids are the bedrock-mantle form and are probed via the OpenAI Responses API on that endpoint; GPT-6 Astra (us./global.), GPT-5.6 Terra and Luna (us./in./global.) and GPT-5.6 Sol @@ -318,6 +331,122 @@ def probe_model(client, model_id: str) -> dict: f"Run 'aws configure', refresh your SSO session, or check the region name."} +RUNTIME_APIS = ("converse", "responses", "chat_completions", "messages", "invoke") + +# Execution lanes Design may emit that this skill has no preflight, evaluation or +# rewrite route for. `mantle_messages` is the Anthropic SDK on the bedrock-mantle +# `/anthropic` endpoint, recorded only with the `workspaces` mantle selector; its +# bare Claude id must not be probed on runtime `/anthropic`, labelled Converse, or +# granted runtime IAM. The lane is deferred, not implemented here. +UNSUPPORTED_LANES = { + "mantle_messages": { + "lane": "Mantle Messages (Anthropic SDK on https://bedrock-mantle.{region}.api.aws/anthropic)", + "selector": "workspaces", + }, +} + + +def unsupported_lane_stop(rows: list, migration_path: str | None = None) -> dict | None: + """Pure: structured stop when any design row (or the plan-level path) selects a + lane this skill does not execute. `rows` are `bedrock_models[]` dicts; each may + carry `migration_path`, `mantle_selector` and `aws_model_id`. Returns None when + every lane is supported.""" + hits = [] + for row in rows: + path = row.get("migration_path") or migration_path + if path in UNSUPPORTED_LANES: + hits.append({"aws_model_id": row.get("aws_model_id"), "migration_path": path, + "mantle_selector": row.get("mantle_selector") + or UNSUPPORTED_LANES[path]["selector"]}) + if not hits and migration_path in UNSUPPORTED_LANES and not rows: + hits.append({"aws_model_id": None, "migration_path": migration_path, + "mantle_selector": UNSUPPORTED_LANES[migration_path]["selector"]}) + if not hits: + return None + path = hits[0]["migration_path"] + info = UNSUPPORTED_LANES[path] + selectors = sorted({h["mantle_selector"] for h in hits if h["mantle_selector"]}) + return { + "ok": False, + "reason": "unsupported_lane", + "lane": path, + "lane_status": "deferred", + "lane_description": info["lane"], + "mantle_selector": selectors[0] if len(selectors) == 1 else selectors, + "failing_models": [h["aws_model_id"] for h in hits], + "rows": hits, + "detail": ( + f"migration_path '{path}' selects the deferred {info['lane']} execution lane " + f"(Design records it only for the '{info['selector']}' mantle selector; this plan's " + f"selector: {', '.join(selectors) or 'absent'}). llm-to-bedrock has no preflight, " + f"evaluation or rewrite route for that lane: do not run the B4 preflight, do not " + f"dispatch Phase C, and do not generate runtime IAM for the bare Claude id. Stop " + f"and tell the user the lane is not yet supported; re-run Design without the " + f"Workspaces requirement to take the supported runtime_messages route."), + } + + +def check_lanes(design_path: str) -> int: + """CLI entry for the lane gate: read aws-design-ai.json, print JSON, exit 1 on stop.""" + try: + with open(design_path, encoding="utf-8") as fh: + design = json.load(fh) + except (OSError, ValueError) as e: + print(json.dumps({"ok": False, "reason": "design_unreadable", + "detail": f"{type(e).__name__}: {e}"}, indent=2)) + return 1 + arch = design.get("ai_architecture", {}) if isinstance(design, dict) else {} + rows = arch.get("bedrock_models") or [] + plan_path = (arch.get("code_migration") or {}).get("migration_path") + stop = unsupported_lane_stop(rows, plan_path) + if stop is not None: + print(json.dumps(stop, indent=2)) + return 1 + print(json.dumps({"ok": True, "reason": "lanes_supported", + "models": [r.get("aws_model_id") for r in rows], + "detail": "Every bedrock_models[] row is on an execution lane this skill supports."}, + indent=2)) + return 0 + + +def parse_targets(specs: list[str]) -> tuple[list[tuple[str, str]], dict | None]: + """Pure: parse `--target ID=API` specs into (model_id, api) pairs. + + Fails CLOSED: a spec without `=API`, or with an API outside RUNTIME_APIS, returns a + structured verdict and no pairs, so the caller probes nothing on a guessed endpoint. + `invoke` is accepted only for embedding ids (the chat lanes never use raw Invoke).""" + pairs: list[tuple[str, str]] = [] + bad: list[tuple[str, str]] = [] + for spec in specs: + for item in spec.split(","): + item = item.strip() + if not item: + continue + model_id, sep, api = item.rpartition("=") + if not sep or not model_id: + bad.append((item, "")) + continue + api = api.strip() + model_id = model_id.strip() + if api not in RUNTIME_APIS or (api == "invoke" and not is_embedding_model(model_id)): + bad.append((model_id, api)) + continue + pairs.append((model_id, api)) + if bad: + return [], { + "ok": False, "reason": "unknown_runtime_api", + "failing_models": [m for m, _ in bad], + "detail": ( + "Every --target must be = with the API from " + "that row (converse | responses | chat_completions | messages; invoke only for an " + "embedding id). Unknown or missing: " + + ", ".join(f"{m}={a}" for m, a in bad) + + ". Preflight fails closed rather than probing a guessed endpoint; resolve the " + "row's API in the saved plan first."), + } + return pairs, None + + def aggregate_failure(results: list) -> dict: """Pure: lift the first failing model's reason/detail to the top level so the orchestrator can branch on a single top-level `reason` (its documented contract).""" @@ -364,21 +493,85 @@ def fetch_bedrock_quotas(region: str) -> list: return [] +def probe_target(client, model_id: str, api: str, region: str, quotas: list, + dataset_size: int) -> dict: + """Probe ONE target on ITS OWN selected API and attach its quota notes.""" + if is_mantle_model(model_id): + verdict = probe_mantle_model(model_id, region) + verdict["model_id"] = model_id + # Mantle enforces per-model input/output TPM quotas and has no RPM + # quota, so an RPM-derived pacing warning would be meaningless here. + verdict["rpm_quota"] = None + verdict["quota_note"] = ( + "Mantle quotas are per-model input/output tokens per minute; there is no RPM " + "quota. Pace on token throughput, and note that prompt-cached input tokens are " + "exempt from the input-TPM quota.") + if model_id.lower() == "openai.gpt-6-astra": + verdict["quota_note"] = ( + f"Verify Astra's mantle quotas in the target account and {region}. " + "Do not assume GPT-5.6's cached-input quota exemption or RPM rules " + "apply; Astra's documented 10x output burndown is runtime-specific.") + return verdict + # Embedding ids never speak a chat API; they are always probed via InvokeModel + # (probe_model), whatever API their row or the plan carries. + if is_embedding_model(model_id): + verdict = probe_model(client, model_id) + elif api == "messages": + verdict = probe_runtime_messages_model(model_id, region) + elif api in ("responses", "chat_completions") and ".openai.gpt-" in model_id \ + and "gpt-oss" not in model_id: + verdict = probe_runtime_openai_model(model_id, region, api) + else: + verdict = probe_model(client, model_id) + verdict["selected_api"] = api + rpm = quota_rpm(quotas, model_id) + verdict["model_id"] = model_id + verdict["rpm_quota"] = rpm + if model_id.rsplit("/", 1)[-1] in ("us.openai.gpt-6-astra", "global.openai.gpt-6-astra"): + verdict["quota_note"] = ( + "Astra runtime TPM usage = input tokens + 10 * output tokens. " + "Check the selected CRIS profile's token quota; an RPM check alone " + "does not establish throughput capacity.") + if rpm is not None and dataset_size > rpm: + verdict["quota_warning"] = ( + f"Dataset ({dataset_size}) exceeds ~{rpm} RPM quota — " + f"Eval will pace with backoff and may be slow.") + return verdict + + def main(argv=None) -> int: ap = argparse.ArgumentParser() - ap.add_argument("--region", required=True) - ap.add_argument("--models", required=True, help="comma-separated model ids") + ap.add_argument("--region") + group = ap.add_mutually_exclusive_group() + group.add_argument("--models", help="comma-separated model ids (legacy single-API form)") + group.add_argument("--target", action="append", default=[], + help="=; repeat per design row") + group.add_argument("--check-lanes", metavar="AWS_DESIGN_AI_JSON", + help="stop with reason unsupported_lane when any bedrock_models[] row " + "selects a deferred lane (mantle_messages); probes nothing") ap.add_argument("--dataset-size", type=int, default=0) ap.add_argument("--runtime-api", choices=["converse", "responses", "chat_completions", "messages"], - default="converse") + default="converse", + help="API applied to every --models id (ignored with --target)") args = ap.parse_args(argv) - model_ids = [m.strip() for m in args.models.split(",") if m.strip()] - if not model_ids: + if args.check_lanes: + return check_lanes(args.check_lanes) + if not args.region or not (args.models or args.target): + ap.error("--region and one of --models / --target are required") + + if args.target: + targets, failure = parse_targets(args.target) + if failure is not None: + print(json.dumps({**failure, "region": args.region, "models": []}, indent=2)) + return 1 + else: + targets = [(m.strip(), args.runtime_api) for m in args.models.split(",") if m.strip()] + if not targets: print(json.dumps({"ok": False, "region": args.region, "models": [], "reason": "no_models", - "detail": "--models resolved to an empty list; nothing to probe."}, + "detail": "--models/--target resolved to an empty list; nothing to probe."}, indent=2)) return 1 @@ -395,44 +588,8 @@ def main(argv=None) -> int: quotas = fetch_bedrock_quotas(args.region) results = [] all_ok = True - for model_id in model_ids: - if is_mantle_model(model_id): - verdict = probe_mantle_model(model_id, args.region) - verdict["model_id"] = model_id - # Mantle enforces per-model input/output TPM quotas and has no RPM - # quota, so an RPM-derived pacing warning would be meaningless here. - verdict["rpm_quota"] = None - verdict["quota_note"] = ( - "Mantle quotas are per-model input/output tokens per minute; there is no RPM " - "quota. Pace on token throughput, and note that prompt-cached input tokens are " - "exempt from the input-TPM quota.") - if model_id.lower() == "openai.gpt-6-astra": - verdict["quota_note"] = ( - f"Verify Astra's mantle quotas in the target account and {args.region}. " - "Do not assume GPT-5.6's cached-input quota exemption or RPM rules " - "apply; Astra's documented 10x output burndown is runtime-specific.") - else: - # Embedding ids in a chat plan's $TARGET_MODELS never speak the selected - # chat API; they are always probed via InvokeModel (probe_model). - if args.runtime_api == "messages" and not is_embedding_model(model_id): - verdict = probe_runtime_messages_model(model_id, args.region) - elif (args.runtime_api != "converse" and ".openai.gpt-" in model_id - and "gpt-oss" not in model_id): - verdict = probe_runtime_openai_model(model_id, args.region, args.runtime_api) - else: - verdict = probe_model(client, model_id) - rpm = quota_rpm(quotas, model_id) - verdict["model_id"] = model_id - verdict["rpm_quota"] = rpm - if model_id.rsplit("/", 1)[-1] in ("us.openai.gpt-6-astra", "global.openai.gpt-6-astra"): - verdict["quota_note"] = ( - "Astra runtime TPM usage = input tokens + 10 * output tokens. " - "Check the selected CRIS profile's token quota; an RPM check alone " - "does not establish throughput capacity.") - if rpm is not None and args.dataset_size > rpm: - verdict["quota_warning"] = ( - f"Dataset ({args.dataset_size}) exceeds ~{rpm} RPM quota — " - f"Eval will pace with backoff and may be slow.") + for model_id, api in targets: + verdict = probe_target(client, model_id, api, args.region, quotas, args.dataset_size) all_ok = all_ok and verdict["ok"] results.append(verdict) diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py index a2293c3de..e93014ed4 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py @@ -14,6 +14,7 @@ is_gpt6_cris_model, is_gpt54_55_cris_model, is_mantle_model, + is_runtime_openai_cris_model, mantle_project_arn, inference_profile_arn, is_inference_profile, @@ -327,17 +328,28 @@ def test_predicates_match_system_profile_arns(self, model_id): [ "openai.gpt-5.4", # bare — mantle, not CRIS "openai.gpt-5.5", - "us.openai.gpt-5.6-sol", # GPT-5.6 cards do not add the grant - "global.openai.gpt-5.6-terra", - "us.openai.gpt-5.4-pro", # not a published runtime id "us.anthropic.claude-sonnet-5", f"arn:aws:bedrock:{REGION}:{ACCOUNT}:inference-profile/us.anthropic.claude-sonnet-5", ], ) - def test_predicates_exclude_bare_mantle_gpt56_and_unrelated_ids(self, model_id): + def test_predicates_exclude_bare_mantle_and_unrelated_ids(self, model_id): assert not is_gpt54_55_cris_model(model_id) assert not requires_default_project_grant(model_id) + @pytest.mark.parametrize( + "model_id", + [ + "us.openai.gpt-5.6-sol", # GPT-5.6 cards (read 2026-11-07) DO add the grant + "global.openai.gpt-5.6-terra", + "us.openai.gpt-5.4-pro", # not a published id, but the id FORM is a runtime GPT CRIS id + ], + ) + def test_family_predicate_excludes_other_gpt_ids_but_id_form_grant_covers_them(self, model_id): + # The GPT-5.4 / GPT-5.5 family predicate stays exact; the default-project grant + # is decided by id form (any runtime GPT CRIS id), not by this family predicate. + assert not is_gpt54_55_cris_model(model_id) + assert requires_default_project_grant(model_id) + @pytest.mark.parametrize("model_id", _GPT54_55_CRIS_SHORT_IDS) def test_generate_policy_grants_project_default_for_short_id(self, model_id): pol = generate_policy([model_id], REGION, ACCOUNT) @@ -368,7 +380,7 @@ def test_bare_mantle_id_gets_mantle_statements_and_no_project_default(self, mode @pytest.mark.parametrize( "model_id", - ["us.anthropic.claude-sonnet-5", "us.openai.gpt-5.6-sol", "global.openai.gpt-5.6-luna"], + ["us.anthropic.claude-sonnet-5", "global.anthropic.claude-opus-5-5", "us.amazon.nova-pro-v1:0"], ) def test_unrelated_runtime_ids_do_not_get_project_default(self, model_id): pol = generate_policy([model_id], REGION, ACCOUNT) @@ -382,3 +394,151 @@ def test_mixed_run_adds_project_default_once(self): ) resources = pol["Statement"][0]["Resource"] assert resources.count(default_project_arn(REGION, ACCOUNT)) == 1 + + +# Every runtime OpenAI-compatible target, by id form (PR 585 review, finding 1). +# Public basis, read 2026-11-07: inference-responses-api.html ("Permissions": a +# Responses request authorizes bedrock:InvokeModel on the inference target AND on +# the account's default project) and the GPT-5.6 Sol / Terra / Luna cards, which +# attach the same note to their runtime CRIS ids (in. exists for Terra and Luna). +_RUNTIME_GPT_CRIS_IDS = [ + "us.openai.gpt-5.6-sol", + "global.openai.gpt-5.6-sol", + "us.openai.gpt-5.6-terra", + "in.openai.gpt-5.6-terra", + "global.openai.gpt-5.6-terra", + "us.openai.gpt-5.6-luna", + "in.openai.gpt-5.6-luna", + "global.openai.gpt-5.6-luna", + "us.openai.gpt-6-astra", + "global.openai.gpt-6-sol", + "us.openai.gpt-6.1-sol", + "us.openai.gpt-5.4", + "global.openai.gpt-5.5", + f"arn:aws:bedrock:{REGION}:{ACCOUNT}:inference-profile/in.openai.gpt-5.6-terra", + f"arn:aws:bedrock:{REGION}:{ACCOUNT}:inference-profile/global.openai.gpt-5.6-sol", + f"arn:aws:bedrock:{REGION}:{ACCOUNT}:inference-profile/us.openai.gpt-5.5", +] + + +class TestDefaultProjectGrantByIdFormAndApi: + def _invoke_resources(self, pol): + return {s["Sid"]: s for s in pol["Statement"]}["BedrockInvokeModelScoped"]["Resource"] + + @pytest.mark.parametrize("model_id", _RUNTIME_GPT_CRIS_IDS) + def test_runtime_gpt_cris_id_form_is_recognized(self, model_id): + assert is_inference_profile(model_id) + assert is_runtime_openai_cris_model(model_id) + assert not is_mantle_model(model_id) + + @pytest.mark.parametrize("model_id", _RUNTIME_GPT_CRIS_IDS) + @pytest.mark.parametrize("runtime_api", [None, "responses", "chat_completions"]) + def test_responses_and_chat_targets_emit_project_default(self, model_id, runtime_api): + pol = generate_policy([model_id], REGION, ACCOUNT, runtime_api=runtime_api) + resources = self._invoke_resources(pol) + short = model_id.rsplit("/", 1)[-1] + base = short.split(".", 1)[1] + assert set(resources) == { + f"arn:aws:bedrock:*::foundation-model/{base}", + inference_profile_arn(model_id, REGION, ACCOUNT), + default_project_arn(REGION, ACCOUNT), + } + assert [s["Sid"] for s in pol["Statement"]] == ["BedrockInvokeModelScoped", "BedrockGetInferenceProfile"] + + @pytest.mark.parametrize("model_id", ["us.openai.gpt-6-astra", "in.openai.gpt-5.6-terra", "us.openai.gpt-5.4"]) + def test_converse_only_route_emits_no_project_default(self, model_id): + # inference-prereq.html (read 2026-11-07): Converse through a profile needs + # bedrock:InvokeModel* on the target + bedrock:GetInferenceProfile; the project + # authorization belongs to Responses creation. + pol = generate_policy([model_id], REGION, ACCOUNT, runtime_api="converse") + resources = self._invoke_resources(pol) + assert default_project_arn(REGION, ACCOUNT) not in resources + assert len(resources) == 2 + assert not requires_default_project_grant(model_id, "converse") + + @pytest.mark.parametrize("model_id", ["openai.gpt-5.6-terra", "openai.gpt-5.6-sol", "openai.gpt-6-sol", "openai.gpt-5.5"]) + @pytest.mark.parametrize("runtime_api", [None, "responses", "chat_completions"]) + def test_bare_mantle_id_never_gets_project_default(self, model_id, runtime_api): + assert not is_runtime_openai_cris_model(model_id) + pol = generate_policy([model_id], REGION, ACCOUNT, runtime_api=runtime_api) + sids = {s["Sid"] for s in pol["Statement"]} + assert "BedrockInvokeModelScoped" not in sids + assert "BedrockMantleInference" in sids + assert default_project_arn(REGION, ACCOUNT) not in json.dumps(pol) + + @pytest.mark.parametrize("model_id", [ + "global.anthropic.claude-sonnet-5", + "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "anthropic.claude-opus-5-5", # runtime in-region form (eu-west-2) + "openai.gpt-oss-120b-1:0", + "us.amazon.nova-lite-v1:0", + ]) + @pytest.mark.parametrize("runtime_api", [None, "responses", "chat_completions", "messages", "converse"]) + def test_anthropic_oss_and_other_runtime_ids_never_get_project_default(self, model_id, runtime_api): + assert not is_runtime_openai_cris_model(model_id) + pol = generate_policy([model_id], "eu-west-2", ACCOUNT, runtime_api=runtime_api) + assert default_project_arn("eu-west-2", ACCOUNT) not in self._invoke_resources(pol) + + def test_mixed_messages_plan_with_gpt_secondary_grants_project_once(self): + # Chat/Responses decides the grant per policy call; the Anthropic primary never + # contributes it and the single GPT target adds it exactly once. + pol = generate_policy(["global.anthropic.claude-sonnet-5", "us.openai.gpt-5.6-sol"], + REGION, ACCOUNT, runtime_api="chat_completions") + resources = self._invoke_resources(pol) + assert resources.count(default_project_arn(REGION, ACCOUNT)) == 1 + assert "arn:aws:bedrock:*::foundation-model/anthropic.claude-sonnet-5" in resources + + def test_unknown_runtime_api_is_rejected(self): + with pytest.raises(ValueError, match="Unknown runtime API"): + generate_policy(["us.openai.gpt-5.6-sol"], REGION, ACCOUNT, runtime_api="invoke") + + def test_us_gov_partition_project_arn(self): + pol = generate_policy(["us-gov.openai.gpt-5.6-terra"], "us-gov-west-1", ACCOUNT, runtime_api="responses") + resources = self._invoke_resources(pol) + assert f"arn:aws-us-gov:bedrock:us-gov-west-1:{ACCOUNT}:project/default" in resources + assert f"arn:aws-us-gov:bedrock:us-gov-west-1:{ACCOUNT}:inference-profile/us-gov.openai.gpt-5.6-terra" in resources + + @pytest.mark.parametrize("runtime_api,expected", [ + ("responses", True), ("chat_completions", True), ("converse", False), + ]) + def test_cli_runtime_api_flag_controls_project_default(self, runtime_api, expected): + result = subprocess.run( # nosec B603 + [sys.executable, str(Path(__file__).parent / "iam_policy.py"), + "--models", "global.openai.gpt-5.6-sol", "--region", REGION, "--account-id", ACCOUNT, + "--runtime-api", runtime_api], + capture_output=True, text=True, + ) + assert result.returncode == 0, result.stderr + policy = json.loads(result.stdout) + assert (default_project_arn(REGION, ACCOUNT) in policy["Statement"][0]["Resource"]) is expected + + +class TestMantleMessagesLaneIsRefused: + """PR 585 review, finding 3: a `mantle_messages` plan (Workspaces selector) has no + execution lane here, and a bare Sonnet id on that path must not receive runtime + `bedrock:InvokeModel*` from the id-only helper.""" + + def test_generate_policy_raises_for_mantle_messages_path(self): + with pytest.raises(ValueError, match="mantle_messages.*deferred execution lane"): + generate_policy(["anthropic.claude-sonnet-5"], REGION, ACCOUNT, + runtime_api="messages", migration_path="mantle_messages") + + def test_runtime_messages_path_still_emits_runtime_policy(self): + pol = generate_policy(["global.anthropic.claude-sonnet-5"], REGION, ACCOUNT, + runtime_api="messages", migration_path="runtime_messages") + assert [s["Sid"] for s in pol["Statement"]] == ["BedrockInvokeModelScoped", "BedrockGetInferenceProfile"] + + def test_cli_mantle_messages_exits_with_structured_reason(self): + result = subprocess.run( # nosec B603 + [sys.executable, str(Path(__file__).parent / "iam_policy.py"), + "--models", "anthropic.claude-sonnet-5", "--region", REGION, "--account-id", ACCOUNT, + "--runtime-api", "messages", "--migration-path", "mantle_messages"], + capture_output=True, text=True, + ) + assert result.returncode == 1 + out = json.loads(result.stdout) + assert out["ok"] is False + assert out["reason"] == "unsupported_lane" + assert out["lane"] == "mantle_messages" + assert "workspaces" in out["detail"] + assert "bedrock:InvokeModel" not in result.stdout.replace(out["detail"], "") diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py index 5fd546889..74d6448c0 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_bedrock.py @@ -372,6 +372,223 @@ def test_main_routes_messages_api_to_the_messages_probe(monkeypatch, capsys): assert json.loads(capsys.readouterr().out)["ok"] is True +class _FakeRuntimeClient: + """Offline stand-in for boto3's bedrock-runtime client.""" + + def __init__(self): + self.calls = [] + self.meta = type("Meta", (), {"region_name": "us-west-2"})() + + def converse(self, **kwargs): + self.calls.append(("converse", kwargs["modelId"])) + return {} + + def invoke_model(self, **kwargs): + self.calls.append(("invoke_model", kwargs["modelId"])) + return {} + + +class _FakeMessagesClient: + """Offline stand-in for anthropic.Anthropic at the runtime /anthropic base URL.""" + calls = [] + + def __init__(self): + self.messages = self + + def create(self, **kwargs): + _FakeMessagesClient.calls.append(("messages", kwargs["model"])) + return {} + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +class _FakeOpenAIClient: + """Offline stand-in for openai.OpenAI at the runtime /openai/v1 base URL.""" + calls = [] + + def __init__(self): + self.chat = type("Chat", (), {"completions": self})() + self.responses = self + + def create(self, **kwargs): + surface = "chat_completions" if "messages" in kwargs else "responses" + _FakeOpenAIClient.calls.append((surface, kwargs["model"])) + return {} + + +@pytest.fixture +def offline_clients(monkeypatch): + """Route every real probe function at offline fake SDK clients and collect calls.""" + import boto3 + runtime = _FakeRuntimeClient() + _FakeMessagesClient.calls = [] + _FakeOpenAIClient.calls = [] + monkeypatch.setattr(boto3, "client", lambda *a, **k: runtime) + monkeypatch.setattr(p, "fetch_bedrock_quotas", lambda region: []) + monkeypatch.setattr(p, "create_messages_client", lambda region, **k: _FakeMessagesClient()) + monkeypatch.setattr(p, "create_openai_client", lambda mid, region, **k: _FakeOpenAIClient()) + monkeypatch.setattr(p, "probe_mantle_model", + lambda *a: (_ for _ in ()).throw(AssertionError("Mantle probe must not run"))) + + def all_calls(): + return runtime.calls + _FakeMessagesClient.calls + _FakeOpenAIClient.calls + return all_calls + + +def test_mixed_claude_messages_and_gpt_chat_each_reach_their_own_probe(offline_clients, capsys): + # PR 585 review, finding 2: B1 collected every aws_model_id and B4 passed one + # --runtime-api, so a Claude Messages primary + GPT Chat secondary POSTed both to + # /anthropic/v1/messages. Each --target row now carries its own API. + import json + rc = p.main(["--region", "us-west-2", + "--target", "global.anthropic.claude-sonnet-5=messages", + "--target", "us.openai.gpt-5.6-sol=chat_completions"]) + out = json.loads(capsys.readouterr().out) + assert rc == 0 and out["ok"] is True + assert sorted(offline_clients()) == sorted([ + ("messages", "global.anthropic.claude-sonnet-5"), + ("chat_completions", "us.openai.gpt-5.6-sol"), + ]) + assert [(m["model_id"], m["selected_api"]) for m in out["models"]] == [ + ("global.anthropic.claude-sonnet-5", "messages"), + ("us.openai.gpt-5.6-sol", "chat_completions"), + ] + + +def test_mixed_claude_messages_nova_converse_and_embeddings_each_reach_their_own_probe( + offline_clients, capsys): + import json + rc = p.main(["--region", "us-west-2", + "--target", "global.anthropic.claude-sonnet-5=messages", + "--target", "us.amazon.nova-lite-v1:0=converse", + "--target", "amazon.titan-embed-text-v2:0=invoke"]) + out = json.loads(capsys.readouterr().out) + assert rc == 0 and out["ok"] is True + assert sorted(offline_clients()) == sorted([ + ("messages", "global.anthropic.claude-sonnet-5"), + ("converse", "us.amazon.nova-lite-v1:0"), + ("invoke_model", "amazon.titan-embed-text-v2:0"), + ]) + + +def test_gpt_responses_row_reaches_the_responses_probe_not_chat(offline_clients, capsys): + import json + rc = p.main(["--region", "us-west-2", + "--target", "global.openai.gpt-6-astra=responses", + "--target", "in.openai.gpt-5.6-terra=chat_completions"]) + assert rc == 0 + assert sorted(offline_clients()) == sorted([ + ("responses", "global.openai.gpt-6-astra"), + ("chat_completions", "in.openai.gpt-5.6-terra"), + ]) + out = json.loads(capsys.readouterr().out) + assert "input tokens + 10 * output tokens" in out["models"][0]["quota_note"] + + +@pytest.mark.parametrize("spec", [ + "global.anthropic.claude-sonnet-5", # no =API + "global.anthropic.claude-sonnet-5=mantle_messages", # a path, not an API + "us.openai.gpt-5.6-sol=invoke", # invoke is not a chat lane + "us.amazon.nova-lite-v1:0=", # empty API +]) +def test_target_with_unknown_api_fails_closed_before_any_probe(spec, offline_clients, capsys): + import json + rc = p.main(["--region", "us-west-2", + "--target", "global.anthropic.claude-sonnet-5=messages", "--target", spec]) + out = json.loads(capsys.readouterr().out) + assert rc == 1 + assert out["ok"] is False + assert out["reason"] == "unknown_runtime_api" + assert out["models"] == [] + assert offline_clients() == [] # nothing probed on a guessed endpoint + + +def test_parse_targets_is_pure_and_fails_closed(): + pairs, failure = p.parse_targets(["a.model=converse,b.embed-v1=invoke", "c.model=messages"]) + assert failure is None + assert pairs == [("a.model", "converse"), ("b.embed-v1", "invoke"), ("c.model", "messages")] + pairs, failure = p.parse_targets(["a.model=converse", "d.model=responses_v2"]) + assert pairs == [] and failure["reason"] == "unknown_runtime_api" + assert failure["failing_models"] == ["d.model"] + + +def test_legacy_models_flag_still_applies_one_api_to_every_id(offline_clients, capsys): + # The single-API form is unchanged for callers that have one row. + rc = p.main(["--region", "us-west-2", "--models", "global.anthropic.claude-sonnet-5", + "--runtime-api", "messages"]) + assert rc == 0 + assert offline_clients() == [("messages", "global.anthropic.claude-sonnet-5")] + + +# --- PR 585 review, finding 3: stop a mantle_messages plan before B4 / Phase C --- + +def _design(rows, plan_path="runtime_messages"): + return {"ai_architecture": {"bedrock_models": rows, + "code_migration": {"migration_path": plan_path}}} + + +def test_unsupported_lane_stop_names_the_deferred_lane_and_workspaces_selector(): + rows = [{"source_model": "claude-sonnet-4-5", "aws_model_id": "anthropic.claude-sonnet-5", + "use_case": "chat", "migration_path": "mantle_messages", "mantle_selector": "workspaces"}] + stop = p.unsupported_lane_stop(rows, "mantle_messages") + assert stop["ok"] is False + assert stop["reason"] == "unsupported_lane" + assert stop["lane"] == "mantle_messages" + assert stop["lane_status"] == "deferred" + assert stop["mantle_selector"] == "workspaces" + assert stop["failing_models"] == ["anthropic.claude-sonnet-5"] + assert "bedrock-mantle.{region}.api.aws/anthropic" in stop["lane_description"] + assert "do not run the B4 preflight" in stop["detail"] + assert "runtime_messages" in stop["detail"] + + +@pytest.mark.parametrize("rows,plan_path", [ + ([{"aws_model_id": "global.anthropic.claude-sonnet-5", "migration_path": "runtime_messages", + "required_client_api": "messages"}], "runtime_messages"), + ([{"aws_model_id": "us.openai.gpt-5.6-sol", "migration_path": "runtime_openai_cris", + "required_client_api": "responses"}, + {"aws_model_id": "amazon.titan-embed-text-v2:0"}], "runtime_openai_cris"), + ([{"aws_model_id": "openai.gpt-6-astra", "migration_path": "mantle_openai_chat", + "mantle_selector": "server_side_tools"}], "mantle_openai_chat"), +]) +def test_supported_lanes_do_not_stop(rows, plan_path): + assert p.unsupported_lane_stop(rows, plan_path) is None + + +def test_check_lanes_cli_stops_on_mantle_messages_and_passes_runtime_messages(tmp_path, capsys): + import json + bad = tmp_path / "bad.json" + bad.write_text(json.dumps(_design( + [{"aws_model_id": "anthropic.claude-sonnet-5", "migration_path": "mantle_messages", + "mantle_selector": "workspaces"}], "mantle_messages"))) + assert p.main(["--check-lanes", str(bad)]) == 1 + out = json.loads(capsys.readouterr().out) + assert out["reason"] == "unsupported_lane" and out["mantle_selector"] == "workspaces" + + good = tmp_path / "good.json" + good.write_text(json.dumps(_design( + [{"aws_model_id": "global.anthropic.claude-sonnet-5", "migration_path": "runtime_messages", + "required_client_api": "messages"}]))) + assert p.main(["--check-lanes", str(good)]) == 0 + out = json.loads(capsys.readouterr().out) + assert out["ok"] is True and out["reason"] == "lanes_supported" + + +def test_check_lanes_uses_plan_level_path_when_rows_omit_it(tmp_path, capsys): + import json + design = tmp_path / "design.json" + design.write_text(json.dumps(_design( + [{"aws_model_id": "anthropic.claude-sonnet-5"}], "mantle_messages"))) + assert p.main(["--check-lanes", str(design)]) == 1 + out = json.loads(capsys.readouterr().out) + assert out["reason"] == "unsupported_lane" + assert out["failing_models"] == ["anthropic.claude-sonnet-5"] + + def test_main_messages_plan_probes_embedding_ids_via_invoke_model(monkeypatch, capsys): # Regression: B1 collects every bedrock_models[].aws_model_id into $TARGET_MODELS and # B4 passes the whole list with one --runtime-api. A runtime_messages plan with a diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_validate_result.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_validate_result.py index 1929bb626..0fd39c65b 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_validate_result.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_validate_result.py @@ -244,6 +244,7 @@ def test_cli_delta_decisions_always_control_ok(tmp_path, capsys): "aws_model_id": "us.anthropic.claude-sonnet-4-6", "use_case": "primary"}], "resolved_model_overrides": {}, + "resolved_runtime_api": "chat_completions", "source_provider": "openai", "source_baseline_available": True, "source_key_sha256": "c" * 64, "log_files": [{"path": "logs/traces.jsonl", "sha256": "d" * 64}], @@ -397,3 +398,28 @@ def test_empty_array_mismatch_detected(tmp_path, capsys): c = write(tmp_path, "c.json", cur) assert vr.main(["--check-run-context", s, "--current", c]) == 1 assert "log_files" in capsys.readouterr().out + + +def test_resolved_runtime_api_round_trips_through_the_resume_gate(tmp_path, capsys): + # PR 585 review, finding 4: SKILL.md saved `resolved_runtime_api` but C0 rebuilt + # current-context.json from a field list that omitted it, so an unchanged resume + # reported MISMATCH $.resolved_runtime_api saved="chat_completions" current="". + # Both contexts now carry the field: an unchanged resume matches, a dropped field or a + # changed API is a single named mismatch for the scoped-invalidation table. + s = write(tmp_path, "saved.json", RUN_CONTEXT) + c = write(tmp_path, "current.json", RUN_CONTEXT) + assert vr.main(["--check-run-context", s, "--current", c]) == 0 + assert capsys.readouterr().out.strip() == "RUN_CONTEXT=match" + + dropped = {k: v for k, v in RUN_CONTEXT.items() if k != "resolved_runtime_api"} + c = write(tmp_path, "current.json", dropped) + assert vr.main(["--check-run-context", s, "--current", c]) == 1 + out = capsys.readouterr().out + assert 'MISMATCH $.resolved_runtime_api saved="chat_completions" current=""' in out + + changed = copy.deepcopy(RUN_CONTEXT) + changed["resolved_runtime_api"] = "responses" + c = write(tmp_path, "current.json", changed) + assert vr.main(["--check-run-context", s, "--current", c]) == 1 + lines = [l for l in capsys.readouterr().out.splitlines() if l.startswith("MISMATCH")] + assert lines == ['MISMATCH $.resolved_runtime_api saved="chat_completions" current="responses"'] diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md index e7ce5f31e..f9c7b04a4 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md @@ -334,9 +334,15 @@ limited to: `aws-bedrock-token-generator`) or a Bedrock API key, **not** an OpenAI key 3. Model ID → the CRIS inference-profile id (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), never the bare `openai.gpt-*` form -4. IAM → `bedrock:InvokeModel*` on the inference-profile and foundation-model ARNs (GPT-6 / GPT-5.5 / GPT-5.4 - also need `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account-id}:project/default`), plus - `bedrock:GetInferenceProfile` on the profile +4. IAM → `bedrock:InvokeModel*` on the inference-profile and foundation-model ARNs, plus + `bedrock:GetInferenceProfile` on the profile, plus — for EVERY runtime Responses / Chat Completions target, + i.e. every proprietary GPT CRIS id above (GPT-5.6 Sol / Terra / Luna, GPT-6, GPT-5.5, GPT-5.4) or its + `inference-profile/` ARN — `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account-id}:project/default`: + a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target + (, "Permissions", read + 2026-11-07), and each model card repeats the requirement for its runtime CRIS ids. + `iam_policy.py --runtime-api ` emits it for `responses` / `chat_completions` rows and + omits it for a `converse` row and for every non-OpenAI id **Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, diff --git a/plugins/aws-startup-advisor/tests/test_ai_only_option_c_framework_route.py b/plugins/aws-startup-advisor/tests/test_ai_only_option_c_framework_route.py index c860fca07..88452cdab 100644 --- a/plugins/aws-startup-advisor/tests/test_ai_only_option_c_framework_route.py +++ b/plugins/aws-startup-advisor/tests/test_ai_only_option_c_framework_route.py @@ -141,4 +141,3 @@ def test_runtime_subroutes_resolve_to_runtime_label() -> None: ) # Mantle label remains for the selector-gated routes. assert "[C] Generate the Mantle migration script and A/B test harness" in section - diff --git a/plugins/aws-startup-advisor/tests/test_runtime_default_review_round1.py b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round1.py new file mode 100644 index 000000000..2c3019e23 --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round1.py @@ -0,0 +1,266 @@ +"""Guards for the PR 585 review-round-1 findings on the bedrock-runtime default. + +Public basis (read 2026-11-07): + https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html + ("Permissions": a Responses request on bedrock-runtime authorizes bedrock:InvokeModel + on the inference target AND on the account's default project) + https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html + https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html + https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html + (each: the identity "also needs bedrock:InvokeModel on your account's default project + ... in addition to the inference profile" for its runtime CRIS ids) + +1. Default-project grant: every setup recipe names the grant for EVERY runtime + Responses / Chat Completions target, GPT-5.6 included, and cites the Responses page. +3. mantle_messages stop: llm-to-bedrock halts a `mantle_messages` plan at a lane gate + before B4 / Phase C instead of labelling it Converse; GCP Generate's Step 0 matches + `mantle_messages` in exactly one branch and keeps the supported `runtime_messages` + route on Messages. +4. resolved_runtime_api: C0's current-context field list carries the field, the + invalidation table has a row for it, B4 retains it on resume, and the actual + comparator (`validate_result.compare_run_contexts`) matches an unchanged resume. + +The id-form / API behaviour of the helpers themselves is tested next to them in +`skills/llm-to-bedrock/scripts/test_iam_policy.py` and `test_preflight_bedrock.py`. +""" +from __future__ import annotations + +import importlib.util +import re +import sys +import types +from pathlib import Path + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +SKILLS = PLUGIN_ROOT / "skills" +SKILL_MD = SKILLS / "llm-to-bedrock" / "SKILL.md" +SCRIPTS = SKILLS / "llm-to-bedrock" / "scripts" +GCP_GENERATE_ARTIFACTS = SKILLS / "gcp-to-aws" / "references" / "phases" / "generate" / "generate-artifacts-ai.md" +GCP_GENERATE_AI = SKILLS / "gcp-to-aws" / "references" / "phases" / "generate" / "generate-ai.md" +AZURE_GENERATE_ARTIFACTS = SKILLS / "azure-to-aws" / "references" / "phases" / "generate" / "generate-artifacts-ai.md" +REPORT_GENERATOR = PLUGIN_ROOT / "agents" / "llm2bedrock-report-generator.md" +CANONICAL_GUIDE = SKILLS / "shared" / "ai" / "ai-openai-to-bedrock.md" +VENDORED_GUIDES = [ + SKILLS / "gcp-to-aws" / "references" / "vendored" / "ai" / "ai-openai-to-bedrock.md", + SKILLS / "azure-to-aws" / "references" / "vendored" / "ai" / "ai-openai-to-bedrock.md", +] +RESPONSES_PAGE = "https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html" + + +def _load(name: str): + """Load a scripts-dir module by path. `validate_result` imports `jsonschema` for its + payload validators; the comparator under test here is pure, so when the plugin test + environment lacks jsonschema a stub stands in for the import only.""" + if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + try: + import jsonschema # noqa: F401 + except ImportError: + sys.modules.setdefault("jsonschema", types.ModuleType("jsonschema")) + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore[union-attr] + return module + + +def _section(text: str, start: str, end: str) -> str: + i = text.index(start) + return text[i: text.index(end, i)] + + +# --- Finding 1: the grant reaches every runtime Responses / Chat target, GPT-5.6 included --- + +def test_setup_recipes_grant_project_default_to_every_runtime_openai_target() -> None: + for path in (CANONICAL_GUIDE, GCP_GENERATE_AI, GCP_GENERATE_ARTIFACTS, AZURE_GENERATE_ARTIFACTS, REPORT_GENERATOR): + text = path.read_text() + windows = [text[max(0, m.start() - 900): m.end() + 900] + for m in re.finditer(r"project/default", text)] + assert windows, f"{path}: no project/default grant sentence" + joined = " ".join(" ".join(w.split()) for w in windows) + assert "GPT-5.6" in joined, f"{path}: the default-project grant does not reach GPT-5.6" + assert re.search(r"Responses\s*/\s*Chat Completions|responses.{0,12}chat_completions", joined), ( + f"{path}: the grant must be tied to the runtime Responses / Chat Completions path") + assert RESPONSES_PAGE in joined, f"{path}: cite the public Responses API page" + assert "2026-11-07" in joined, f"{path}: record the read date" + # Not a GPT-6-only or GPT-6 + 5.4/5.5-only grant any more. + assert not re.search(r"For (a )?GPT-6 CRIS( id)?, and for the GPT-5\.5 / GPT-5\.4 CRIS ids", text), path + assert "GPT-6 / GPT-5.5 / GPT-5.4\n also need" not in text, path + + +def test_vendored_openai_guides_match_canonical() -> None: + for vendored in VENDORED_GUIDES: + assert vendored.read_bytes() == CANONICAL_GUIDE.read_bytes(), vendored + + +# --- Finding 3: mantle_messages stops before B4 / Phase C; Step 0 branches are disjoint --- + +def test_skill_has_a_lane_gate_before_b4_and_phase_c() -> None: + text = SKILL_MD.read_text() + b1 = _section(text, "### B1 — Read Assess outputs", "### B2 — AWS identity confirmation") + assert "--check-lanes" in b1, "B1 must run preflight_bedrock.py --check-lanes" + assert "unsupported_lane" in b1 + assert "mantle_messages" in b1 and "workspaces" in b1 + assert "Mantle Messages" in b1 and "deferred" in b1 + assert "runtime_messages" in b1, "the stop must point at the supported route" + assert "Do NOT run B4" in b1 and "do NOT dispatch C1" in b1 + # The context rule no longer passes mantle_messages through as a Converse plan. + context = _section(text, "### The context block", "### C0 — Run-context gate") + assert "`mantle_messages` pass through `normalize_api_path` unchanged" not in context + assert "mantle_messages` never reaches this" in context + assert "must not be labelled" in context + surface = next(line for line in context.splitlines() if line.startswith("Target API surface:")) + assert "mantle_messages never reaches a dispatch" in surface + + +def test_iam_helper_refuses_mantle_messages_for_a_bare_claude_id() -> None: + iam = _load("iam_policy") + try: + iam.generate_policy(["anthropic.claude-sonnet-5"], "us-east-1", "123456789012", + runtime_api="messages", migration_path="mantle_messages") + except ValueError as e: + assert "mantle_messages" in str(e) and "workspaces" in str(e) + else: # pragma: no cover - the assertion above is the contract + raise AssertionError("generate_policy emitted runtime permissions for a mantle_messages plan") + + +def test_preflight_lane_gate_is_structured_and_names_lane_and_selector() -> None: + preflight = _load("preflight_bedrock") + stop = preflight.unsupported_lane_stop( + [{"aws_model_id": "anthropic.claude-sonnet-5", "migration_path": "mantle_messages", + "mantle_selector": "workspaces"}], "mantle_messages") + assert stop["reason"] == "unsupported_lane" + assert stop["lane"] == "mantle_messages" and stop["lane_status"] == "deferred" + assert stop["mantle_selector"] == "workspaces" + assert preflight.unsupported_lane_stop( + [{"aws_model_id": "global.anthropic.claude-sonnet-5", "migration_path": "runtime_messages", + "required_client_api": "messages"}], "runtime_messages") is None + + +def _step0_bullets(text: str) -> list[str]: + step0 = text.split("## Step 0: Determine Artifact Path", 1)[1].split("## Step 1M:", 1)[0] + return [line for line in step0.splitlines() if line.startswith("- ")] + + +def _bullet_matches(bullet: str, migration_path: str) -> bool: + """Evaluate the Step 0 bullet heads the way an agent reads them: an exact-value + bullet matches only its listed values; the `starts with "mantle"` bullet is scoped + to the OpenAI-SDK spellings it enumerates.""" + head = bullet.split("→", 1)[0] + excluded = set(re.findall(r'`"([a-z_\-]+)"` is NOT this branch', head)) + listed = set(re.findall(r'`"([a-z_\-]+)"`', head)) - excluded + if "starts with" in head: + return migration_path.startswith("mantle") and migration_path in listed + if "or absent" in head: + return migration_path in listed or migration_path == "" + return migration_path in listed + + +def test_gcp_step0_matches_mantle_messages_in_exactly_one_branch() -> None: + bullets = _step0_bullets(GCP_GENERATE_ARTIFACTS.read_text()) + matches = [b for b in bullets if _bullet_matches(b, "mantle_messages")] + assert len(matches) == 1, [m[:120] for m in matches] + branch = matches[0] + assert '`migration_path` = `"mantle_messages"`' in branch + assert "bedrock-mantle.{region}.api.aws/anthropic" in branch + assert "`bedrock-mantle:*` IAM" in branch + assert "Do NOT generate `migrate_to_mantle.sh`" in branch + # The broad Mantle branch now enumerates the OpenAI-SDK spellings and excludes Messages. + mantle_branch = next(b for b in bullets if 'starts with `"mantle"`' in b) + assert '`"mantle_messages"` is NOT this branch' in mantle_branch + for spelling in ("mantle", "mantle_openai_chat", "mantle_openai_responses"): + assert [b for b in bullets if _bullet_matches(b, spelling)] == [mantle_branch], spelling + + +def test_gcp_step0_supported_runtime_messages_route_keeps_messages() -> None: + bullets = _step0_bullets(GCP_GENERATE_ARTIFACTS.read_text()) + matches = [b for b in bullets if _bullet_matches(b, "runtime_messages")] + assert len(matches) == 1 + branch = matches[0] + assert "bedrock-runtime.{region}.amazonaws.com/anthropic" in branch + assert '`required_client_api` is `"messages"`' in branch + assert "do NOT rewrite to Converse" in branch + assert "do NOT generate a Mantle script" in branch + + +# --- Finding 4: resolved_runtime_api survives the resume gate --- + +def _c0_field_names(text: str) -> list[str]: + c0 = _section(text, "### C0 — Run-context gate", "### C1 — Analyzer") + template = re.search(r"```json\n(.*?)\n```", c0, re.S).group(1) + return re.findall(r'^\s*"([a-z_]+)":', template, re.M) + + +def test_c0_context_schema_and_invalidation_table_carry_resolved_runtime_api() -> None: + text = SKILL_MD.read_text() + fields = _c0_field_names(text) + assert "resolved_runtime_api" in fields, fields + c0 = _section(text, "### C0 — Run-context gate", "### C1 — Analyzer") + table = c0.split("| Mismatched field(s)")[1] + row = next(line for line in table.splitlines() if line.startswith("| resolved_runtime_api")) + cells = [c.strip() for c in row.strip("|").split("|")] + assert cells[1] == "EVAL, REWRITE, REPORT", row + assert cells[2] == "ANALYSIS, INGESTION", row + + +def test_b4_resolves_the_api_once_and_reads_it_back_on_resume() -> None: + text = SKILL_MD.read_text() + b4 = _section(text, "### B4 — Bedrock preflight", "## Phase C — Execute") + assert "$RESOLVED_RUNTIME_API" in b4 + assert "run-context.json" in b4 and "resolved_runtime_api" in b4 + assert "reuse it without" in b4 and "asking" in b4 + assert "--target =" in b4 + assert "unknown_runtime_api" in b4 + context = _section(text, "### The context block", "### C0 — Run-context gate") + assert "`plan_runtime_api` is B4's `$RESOLVED_RUNTIME_API`" in context + assert "written to current-context.json as\n`resolved_runtime_api`" in context + + +def _context_from_field_list(fields: list[str], resolved_runtime_api: str | None) -> dict: + """Build a context with exactly C0's documented field list and plain fixture values.""" + ctx: dict = {} + for name in fields: + if name == "target_models": + ctx[name] = [{"source_model": "gpt-5.5", "aws_model_id": "us.openai.gpt-5.5", "use_case": "chat"}] + elif name in ("resolved_model_overrides",): + ctx[name] = {} + elif name == "log_files": + ctx[name] = [] + elif name == "source_baseline_available": + ctx[name] = False + elif name in ("max_golden_cases", "schema_version", "evaluation_contract_version"): + ctx[name] = 200 if name == "max_golden_cases" else 3 if name.startswith("evaluation") else 1 + elif name == "resolved_runtime_api": + if resolved_runtime_api is not None: + ctx[name] = resolved_runtime_api + else: + ctx[name] = f"fixture-{name}" + return ctx + + +def test_actual_comparator_matches_an_unchanged_resume_with_resolved_runtime_api() -> None: + validate = _load("validate_result") + fields = _c0_field_names(SKILL_MD.read_text()) + saved = _context_from_field_list(fields, "chat_completions") + current = _context_from_field_list(fields, "chat_completions") + assert validate.compare_run_contexts(saved, current) == [] + + +def test_actual_comparator_reproduces_the_reviewers_mismatch_when_the_field_is_dropped() -> None: + # The finding: a current-context built from a field list that omits the saved answer + # mismatches on $.resolved_runtime_api (saved="chat_completions" current=""). + validate = _load("validate_result") + fields = _c0_field_names(SKILL_MD.read_text()) + saved = _context_from_field_list(fields, "chat_completions") + current = _context_from_field_list(fields, None) + lines = validate.compare_run_contexts(saved, current) + assert lines == ['MISMATCH $.resolved_runtime_api saved="chat_completions" current=""'] + + +def test_actual_comparator_flags_a_changed_api_for_scoped_invalidation() -> None: + validate = _load("validate_result") + fields = _c0_field_names(SKILL_MD.read_text()) + saved = _context_from_field_list(fields, "chat_completions") + current = _context_from_field_list(fields, "responses") + lines = validate.compare_run_contexts(saved, current) + assert len(lines) == 1 and lines[0].startswith("MISMATCH $.resolved_runtime_api ") + assert validate.top_key("$.resolved_runtime_api") == "resolved_runtime_api" From 4ff48f5ad505cefd19e9a9cc466f44ddfd5084c2 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 7 Nov 2026 20:10:22 -0800 Subject: [PATCH 13/41] fix(aws-startup-advisor): grant the default project per bedrock_models[] row in iam_policy (--target ID=API) so a GPT secondary under a Claude Messages primary keeps it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review-gate finding B1 (pr585-review-round-1): the §6.4 report-generator recipe passed the primary's single `Target API surface` as --runtime-api for every id, and generate_policy applied it to all ids, so a runtime_messages plan with a GPT Chat/Responses secondary emitted no arn:...:project/default for that secondary (a regression against 6ca403b6's id-only call). - iam_policy.py: new repeatable `--target =` form (parse_targets, fails closed with reason unknown_runtime_api before any policy is built) and a `target_apis` kwarg on generate_policy; each row's own API decides requires_default_project_grant for that id, overriding the plan-wide runtime_api. `--models --runtime-api` stays as the legacy single-API form; --models/--target are mutually exclusive; the mantle_messages lane stop still wins over every other check. - llm2bedrock-report-generator.md §6.4: recipe passes one --target pair per bedrock_models[] row read from aws-design-ai.json (primary row = Target API surface; `invoke` for embedding rows), explicitly forbids applying the primary's surface to other rows, reports unknown_runtime_api verbatim, and documents the legacy form as single-row only. - Shared guide IAM step, GCP generate-ai.md / generate-artifacts-ai.md and Azure generate-artifacts-ai.md recipes name the per-row form; both vendored copies of the shared guide re-synced (byte-identical). - Tests: test_iam_policy.py TestPerRowTargetApis (reviewer's spot-check as the regression; Claude/messages + GPT/chat -> project/default exactly once; Claude/messages + Nova/converse + embeddings/invoke -> absent; GPT chat row beside GPT converse row -> once; row API overrides plan-wide; fail-closed parse; CLI --target grant / unknown API exit 1 / mutual exclusion). tests/test_runtime_default_review_round1.py: text contract pinning the §6.4 recipe (--target per row, no --runtime-api in the command, per-row warning, unknown_runtime_api) and the four setup recipes; mixed plan driven through parse_targets + generate_policy. Validation (run in this worktree, mise python 3.12): - mise -C plugins/aws-startup-advisor run ci: exit 0 -- test 1431 passed + 222 subtests; test:telemetry 311 passed; vendored parity 64 PASS - uv run --project skills/llm-to-bedrock/scripts pytest skills/llm-to-bedrock: 614 passed (bare mise python without the uv project: 603 passed, 11 failed on ModuleNotFoundError httpx in test_astra_execution_contract -- environment, not code) - pytest skills/agent-advisor: 570 passed (no agent-advisor files touched) - pytest tests/test_gpt54_55_cris_prefix_scope.py: 3 passed - python3 tools/validate.py: All validations passed - tools/sync-vendored.py --check: PASS, 64 byte-identical - git diff --check: clean - mise run build: gitleaks fails only on pre-existing upstream commit f8d09fd1 (tests/test_bedrock_migration_assessment_discover.py:118 generic-api-key); unrelated to this branch --- .../agents/llm2bedrock-report-generator.md | 23 ++-- .../phases/generate/generate-artifacts-ai.md | 8 +- .../vendored/ai/ai-openai-to-bedrock.md | 5 +- .../references/phases/generate/generate-ai.md | 2 +- .../phases/generate/generate-artifacts-ai.md | 2 +- .../vendored/ai/ai-openai-to-bedrock.md | 5 +- .../llm-to-bedrock/scripts/iam_policy.py | 94 ++++++++++++--- .../llm-to-bedrock/scripts/test_iam_policy.py | 109 ++++++++++++++++++ .../skills/shared/ai/ai-openai-to-bedrock.md | 5 +- .../test_runtime_default_review_round1.py | 35 ++++++ 10 files changed, 255 insertions(+), 33 deletions(-) diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md index 119f7f97a..210f47447 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md @@ -565,19 +565,28 @@ Generate a least-privilege IAM policy scoped to the exact model ARNs selected du ```bash uv run --project python /iam_policy.py \ - --models \ + --target = [--target ... one per bedrock_models[] row] \ --region \ --account-id \ - --runtime-api \ --migration-path \ --output /.saws-migrate/iam-policy.json ``` -Omit `--runtime-api` only when the context carries no API surface (id-only behaviour). A `mantle_messages` -path never reaches this agent (the orchestrator's lane gate stops it before Phase C); if it somehow does, the -helper exits 1 with `reason: unsupported_lane` — report that verbatim and do not hand-write a policy. - -The script handles the dual-ARN pattern: foundation-model ARNs for plain model IDs and inference-profile ARNs for geo-prefixed IDs (e.g. `us.anthropic.claude-sonnet-4-6`). For any cross-region (CRIS) target it also emits a `BedrockGetInferenceProfile` statement (`bedrock:GetInferenceProfile` on the profile ARNs) — the documented prerequisite for invoking through a profile (, read 2026-11-07). For every runtime Responses / Chat Completions target — any proprietary GPT CRIS id by id form (GPT-5.6 Sol / Terra / Luna, the GPT-6 family, GPT-5.5 / GPT-5.4: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`, or their `inference-profile/` ARNs) when `--runtime-api` is `responses` or `chat_completions` (or omitted) — it adds a third resource, `arn:aws:bedrock:::project/default`, because a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, read 2026-11-07) and each of those model cards (read 2026-11-07) repeats the requirement; do not strip it from the report's policy excerpt. A `converse` surface and every Anthropic / Nova / gpt-oss id get no project resource. The output is a ready-to-use IAM policy JSON file. +**One `--target` pair per row, each with ITS OWN API.** Read `ai_architecture.bedrock_models[]` from +`/aws-design-ai.json` and emit `=` for every row +(`responses` | `chat_completions` | `converse` | `messages`; `invoke` for an embedding row). The primary chat +row's API is the `Target API surface` from your context (the value B4 resolved; use it when that row records +no `required_client_api`). Do NOT apply the primary's surface to the other rows: the grant is decided per row, +so a `runtime_messages` plan whose secondary is a GPT Chat / Responses CRIS id still gets `project/default` +for that secondary, while a Nova Converse secondary does not. If any row's API is missing or unknown the +helper exits 1 with `reason: unknown_runtime_api` and writes no policy — report that verbatim, do not +substitute an API and do not hand-write a policy. The legacy single-API form +`--models --runtime-api ` is still accepted but applies ONE API to +every id; use it only for a single-row plan. A `mantle_messages` path never reaches this agent (the +orchestrator's lane gate stops it before Phase C); if it somehow does, the helper exits 1 with +`reason: unsupported_lane` — report that verbatim and do not hand-write a policy. + +The script handles the dual-ARN pattern: foundation-model ARNs for plain model IDs and inference-profile ARNs for geo-prefixed IDs (e.g. `us.anthropic.claude-sonnet-4-6`). For any cross-region (CRIS) target it also emits a `BedrockGetInferenceProfile` statement (`bedrock:GetInferenceProfile` on the profile ARNs) — the documented prerequisite for invoking through a profile (, read 2026-11-07). For every runtime Responses / Chat Completions target — any proprietary GPT CRIS id by id form (GPT-5.6 Sol / Terra / Luna, the GPT-6 family, GPT-5.5 / GPT-5.4: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`, or their `inference-profile/` ARNs) whose own `--target` API is `responses` or `chat_completions` (legacy form: `--runtime-api` set to one of those, or omitted) — it adds a third resource, `arn:aws:bedrock:::project/default`, because a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, read 2026-11-07) and each of those model cards (read 2026-11-07) repeats the requirement; do not strip it from the report's policy excerpt. A row on `converse` or `messages` and every Anthropic / Nova / gpt-oss / embedding id get no project resource. The output is a ready-to-use IAM policy JSON file. **Runtime OpenAI and Runtime Messages lanes need no Mantle statement.** A `Rewrite strategy: runtime_openai` or `Rewrite strategy: runtime_messages` run keeps the source SDK but authorizes through `bedrock:InvokeModel*` (+ `GetInferenceProfile`, and the default-project ARN for every runtime Responses / Chat Completions GPT CRIS id above) on the `bedrock-runtime` endpoint; report exactly the statements the script produced and do not add `bedrock-mantle:*` for them. Per-token model pricing is the same on both endpoints (, read 2026-11-07), so the endpoint choice does not change the §6.3 cost table. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md index bdd9e8d99..feb86e8d5 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -98,9 +98,11 @@ their `inference-profile/` ARNs — the IAM policy must grant `bedrock:InvokeMod ARN: a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, "Permissions", read 2026-11-07), and each of those model cards (read 2026-11-07) repeats the -requirement for its runtime CRIS ids. `iam_policy.py --runtime-api ` emits -the third ARN for exactly those rows and omits it for a `converse` row (the Converse-only -application-inference-profile route) and for every Anthropic / Nova / gpt-oss id. The mantle +requirement for its runtime CRIS ids. `iam_policy.py --target =` +(one pair per `bedrock_models[]` row, each with its own API — a GPT Chat / Responses secondary beside +a Claude Messages primary still gets the grant) emits the third ARN for exactly those rows and omits +it for a `converse` or `messages` row (Converse is the application-inference-profile route) and for +every Anthropic / Nova / gpt-oss id. The mantle `bedrock-mantle:*` policy (bare `openai.gpt-5.5` / `openai.gpt-5.4` / `openai.gpt-5.6-*` / `openai.gpt-6-*` ids included) is unaffected. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index f9c7b04a4..8e874d3eb 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -341,8 +341,9 @@ limited to: a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, "Permissions", read 2026-11-07), and each model card repeats the requirement for its runtime CRIS ids. - `iam_policy.py --runtime-api ` emits it for `responses` / `chat_completions` rows and - omits it for a `converse` row and for every non-OpenAI id + `iam_policy.py --target =` (one pair per `bedrock_models[]` row, each + row's own API) emits it for `responses` / `chat_completions` rows and omits it for a `converse` or + `messages` row and for every non-OpenAI id — a GPT secondary beside a Claude Messages primary still gets it **Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md index 08832c532..4fd7ebe74 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md @@ -30,7 +30,7 @@ Check `preferences.json` → `ai_constraints.ai_framework` to determine the migr **OpenAI SDK users keeping the same model (config path for a Responses source; small reshape for a Chat Completions source)** — `ai_framework` = `direct` AND `ai_source` = `openai` AND (`migration_path` == `"runtime_openai_cris"` (the default) OR `migration_path` **starts with** `mantle` (`mantle`, `mantle_openai_responses`, `mantle_openai_chat`, selector-gated)): -- `runtime_openai_cris` (the default — GPT-5.6/GPT-6/GPT-5.5/GPT-5.4 via `bedrock-runtime` CRIS): point the client at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` — **not** the mantle URL. Update the model string to the CRIS inference-profile ID the selected model's card lists (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), not the bare ID. Grant standard `bedrock:InvokeModel*` actions against the CRIS inference-profile ARN — **not** `bedrock-mantle:*`. For every runtime Responses / Chat Completions target — any proprietary GPT CRIS id by id form: GPT-5.6 Sol / Terra / Luna, the GPT-6 family, and GPT-5.5 / GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` alongside the inference-profile ARN: the Responses API authorizes the default project separately from the inference target (, read 2026-11-07) and each model card repeats it for its runtime CRIS ids; `iam_policy.py --runtime-api responses|chat_completions` emits this third ARN for exactly those ids, and omits it for a `converse` row. Read `aws-design-ai.json` → `ai_architecture.bedrock_models[].required_client_api` — the API the source app already calls — and emit that surface (`responses.create`, `chat.completions.create`, or Converse when Design recorded `converse` for a Guardrails or application-inference-profile need); for a region-only move it is simply the source's own API. +- `runtime_openai_cris` (the default — GPT-5.6/GPT-6/GPT-5.5/GPT-5.4 via `bedrock-runtime` CRIS): point the client at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` — **not** the mantle URL. Update the model string to the CRIS inference-profile ID the selected model's card lists (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), not the bare ID. Grant standard `bedrock:InvokeModel*` actions against the CRIS inference-profile ARN — **not** `bedrock-mantle:*`. For every runtime Responses / Chat Completions target — any proprietary GPT CRIS id by id form: GPT-5.6 Sol / Terra / Luna, the GPT-6 family, and GPT-5.5 / GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` alongside the inference-profile ARN: the Responses API authorizes the default project separately from the inference target (, read 2026-11-07) and each model card repeats it for its runtime CRIS ids; `iam_policy.py --target =` (one pair per row, each row's own API) emits this third ARN for exactly the `responses` / `chat_completions` rows — a GPT secondary beside a Claude Messages primary included — and omits it for a `converse` or `messages` row. Read `aws-design-ai.json` → `ai_architecture.bedrock_models[].required_client_api` — the API the source app already calls — and emit that surface (`responses.create`, `chat.completions.create`, or Converse when Design recorded `converse` for a Guardrails or application-inference-profile need); for a region-only move it is simply the source's own API. - `mantle` / `mantle_openai_responses` / `mantle_openai_chat` (only when Design recorded a `mantle_selector`, see `references/vendored/ai/bedrock-endpoint-policy.md`): point the client at `https://bedrock-mantle.{region}.api.aws/openai/v1` — note the `openai/v1` segment for proprietary GPT models; a bare `/v1` 404s. Swap the credential for a Bedrock API key or the auto-refreshing token provider — **not** the existing OpenAI key. Update the model string to the bare Bedrock model ID (`openai.gpt-*` for a same-model move). Grant the `bedrock-mantle:*` actions; `bedrock:InvokeModel` does not authorize these models. - **Config-only if the source already calls `responses.create`.** A Chat Completions source for GPT-6 or GPT-5.6 may keep `chat.completions.create`/`choices[0].message` parsing unchanged — Chat Completions is supported on both mantle and runtime CRIS for these models, so reshape to Responses is optional (only for a Responses-only feature or demonstrated constraint). GPT-5.5/GPT-5.4 follow the same rule — their model cards (read 2026-11-06) list Chat Completions on both mantle and runtime, so the reshape is optional for them too; probe the selected API against the target account before committing, as for any model. - Test in staging, validate responses diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md index 905cfb285..2073593ea 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -215,7 +215,7 @@ Generate `ai-migration/setup_bedrock.sh`. - Dry-run by default (`--execute` flag to run for real) - Step 1 — Request model access: List each model from `aws-design-ai.json` → `bedrock_models[].aws_model_id` and the embedding model -- Step 2 — Create IAM role: Trust policy for the compute platform (Lambda, ECS, or EC2 based on `aws-design.json` if present). Match inference permissions to the selected endpoint. Mantle needs `bedrock-mantle:CreateInference` scoped to `arn:aws:bedrock-mantle:{region}:{account_id}:project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` when using bearer credentials. Runtime needs `bedrock:InvokeModel` / `bedrock:InvokeModelWithResponseStream` against the selected foundation-model ARN and, for CRIS, the exact inference-profile ARN. For EVERY `runtime_openai_cris` row whose `required_client_api` is `responses` or `chat_completions` — classified by id form, not model family: any `us.`/`global.` (and, for GPT-5.6 Terra and Luna, `in.`) `openai.gpt-*` CRIS id, so GPT-5.6 Sol / Terra / Luna, the GPT-6 family, GPT-5.5 and GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default`: a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, "Permissions", read 2026-11-07), and each of those model cards (read 2026-11-07) repeats the requirement for its runtime CRIS ids. `iam_policy.py --runtime-api ` emits that third ARN for exactly those rows. A row with `required_client_api: "converse"` (the Converse-only application-inference-profile route) and every Anthropic / Nova / gpt-oss id get no such grant; a bare `openai.gpt-5.5` / `openai.gpt-5.4` / `openai.gpt-5.6-*` / `openai.gpt-6-*` id is a mantle target and gets no such grant. Runtime actions do not authorize Mantle. +- Step 2 — Create IAM role: Trust policy for the compute platform (Lambda, ECS, or EC2 based on `aws-design.json` if present). Match inference permissions to the selected endpoint. Mantle needs `bedrock-mantle:CreateInference` scoped to `arn:aws:bedrock-mantle:{region}:{account_id}:project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` when using bearer credentials. Runtime needs `bedrock:InvokeModel` / `bedrock:InvokeModelWithResponseStream` against the selected foundation-model ARN and, for CRIS, the exact inference-profile ARN. For EVERY `runtime_openai_cris` row whose `required_client_api` is `responses` or `chat_completions` — classified by id form, not model family: any `us.`/`global.` (and, for GPT-5.6 Terra and Luna, `in.`) `openai.gpt-*` CRIS id, so GPT-5.6 Sol / Terra / Luna, the GPT-6 family, GPT-5.5 and GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default`: a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, "Permissions", read 2026-11-07), and each of those model cards (read 2026-11-07) repeats the requirement for its runtime CRIS ids. `iam_policy.py --target =` (one pair per `bedrock_models[]` row, each row's own API, so a GPT Chat / Responses secondary beside a Claude Messages primary still gets it) emits that third ARN for exactly those rows. A row with `required_client_api: "converse"` (the Converse-only application-inference-profile route) and every Anthropic / Nova / gpt-oss id get no such grant; a bare `openai.gpt-5.5` / `openai.gpt-5.4` / `openai.gpt-5.6-*` / `openai.gpt-6-*` id is a mantle target and gets no such grant. Runtime actions do not authorize Mantle. - If the runtime client uses `BEDROCK_INFERENCE_PROFILE_ARN` from the generated monitoring output, also authorize that exact `application-inference-profile/…` ARN, retaining the backing model/CRIS resources needed for the selected model and fallback. Resolve it from the same output used by the client; do not grant every application profile. Re-run setup after enabling or changing this override so the effective invocation target and IAM remain aligned. See [inference-profile IAM prerequisites](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-prereq.html). - Compliance carry-through: if `preferences.json` → `design_constraints.compliance` is set and not `none`/`unknown` (full-flow Q2 / AI-only Q1.5), the script and its printed notes MUST reflect the design's Step 0.7 constraints — `AWS_REGION` from the compliance-constrained region (GovCloud for fedramp, EU for gdpr), `eu.` inference-profile model IDs for gdpr, and for hipaa a printed warning that Bedrock invocation logs retain original content (KMS-encrypt + restrict IAM on the log group before enabling) - Step 3 — Print required environment variables: `AWS_REGION`, `AI_PROVIDER=bedrock`, model IDs diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index f9c7b04a4..8e874d3eb 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -341,8 +341,9 @@ limited to: a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, "Permissions", read 2026-11-07), and each model card repeats the requirement for its runtime CRIS ids. - `iam_policy.py --runtime-api ` emits it for `responses` / `chat_completions` rows and - omits it for a `converse` row and for every non-OpenAI id + `iam_policy.py --target =` (one pair per `bedrock_models[]` row, each + row's own API) emits it for `responses` / `chat_completions` rows and omits it for a `converse` or + `messages` row and for every non-OpenAI id — a GPT secondary beside a Claude Messages primary still gets it **Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py index 63624ff6d..091a14f3f 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py @@ -14,6 +14,9 @@ # OpenAI-compatible pair is what the default-project grant keys on. RUNTIME_APIS = frozenset({"converse", "responses", "chat_completions", "messages"}) _OPENAI_COMPATIBLE_APIS = frozenset({"responses", "chat_completions"}) +# A per-row `--target ID=API` spec may also say `invoke` for an embedding row (the +# value preflight accepts for those rows); it never selects the OpenAI-compatible path. +TARGET_APIS = RUNTIME_APIS | {"invoke"} # Execution lanes this helper refuses to authorize. `mantle_messages` (Anthropic SDK # on the bedrock-mantle `/anthropic` endpoint, selected by the `workspaces` mantle @@ -155,14 +158,46 @@ def requires_default_project_grant(model_id: str, runtime_api: str | None = None return runtime_api in _OPENAI_COMPATIBLE_APIS +def parse_targets(specs: list[str]) -> list[tuple[str, str]]: + """Pure: parse repeatable `--target ID=API` specs (comma-joined items allowed) + into (model_id, api) pairs, one per `bedrock_models[]` row. + + Fails CLOSED: a spec without `=API`, or with an API outside TARGET_APIS, raises + `ValueError` before any policy is built, so a row whose API the plan does not + record is never granted (or denied) the default project on a guess.""" + pairs: list[tuple[str, str]] = [] + bad: list[str] = [] + for spec in specs: + for item in spec.split(","): + item = item.strip() + if not item: + continue + model_id, sep, api = item.rpartition("=") + model_id, api = model_id.strip(), api.strip() + if not sep or not model_id or api not in TARGET_APIS: + bad.append(item if sep else f"{item}=") + continue + pairs.append((model_id, api)) + if bad: + raise ValueError( + "Every --target must be = with the API from that " + f"row ({' | '.join(sorted(TARGET_APIS))}). Unknown or missing: {', '.join(bad)}. " + "Resolve the row's API in the saved plan instead of guessing it.") + return pairs + + def generate_policy(model_ids: list[str], region: str, account_id: str, runtime_api: str | None = None, - migration_path: str | None = None) -> dict: + migration_path: str | None = None, + target_apis: dict[str, str] | None = None) -> dict: """Build a scoped IAM policy covering exactly the given model IDs. `runtime_api` is the plan's selected API (`responses`, `chat_completions`, `converse`, `messages`); it decides the default-project grant (see `requires_default_project_grant`). None keeps the id-only behaviour. + `target_apis` maps a model id to ITS OWN row's API and takes precedence over + `runtime_api` for that id, so a mixed plan (Claude Messages primary + GPT Chat + secondary) grants the default project to the GPT row and not to the Claude row. `migration_path` is the plan's `code_migration.migration_path`; an UNSUPPORTED lane (`mantle_messages`) raises `ValueError` instead of emitting runtime `bedrock:InvokeModel*` for a bare Claude id that the lane would never @@ -193,6 +228,11 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, if runtime_api is not None and runtime_api not in RUNTIME_APIS: raise ValueError(f"Unknown runtime API {runtime_api!r}; expected one of " f"{sorted(RUNTIME_APIS)}") + target_apis = dict(target_apis or {}) + for mid, api in target_apis.items(): + if api not in TARGET_APIS: + raise ValueError(f"Unknown runtime API {api!r} for target {mid!r}; expected one of " + f"{sorted(TARGET_APIS)}") runtime_ids = [m for m in model_ids if not is_mantle_model(m, region)] mantle_ids = [m for m in model_ids if is_mantle_model(m, region)] @@ -209,8 +249,8 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, # InvokeModel on the inference-profile ARN AND the default project # ARN; both are required, neither replaces the other. Converse-only # routes, Anthropic ids and every other runtime target keep the - # 2-ARN shape above. - if requires_default_project_grant(mid, runtime_api): + # 2-ARN shape above. The row's own API wins over the plan-wide one. + if requires_default_project_grant(mid, target_apis.get(mid, runtime_api)): resources.append(default_project_arn(region, account_id)) else: resources.append(foundation_model_arn(mid, region)) @@ -268,30 +308,54 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, import argparse parser = argparse.ArgumentParser(description="Generate scoped Bedrock IAM policy") - parser.add_argument("--models", required=True, help="Comma-separated model IDs") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--models", help="Comma-separated model IDs (legacy single-API form)") + group.add_argument("--target", action="append", default=[], + help="=; repeat once per " + "bedrock_models[] row. Each row's API decides its own " + "default-project grant (responses/chat_completions add it for a " + "runtime GPT CRIS id; converse/messages/invoke do not). An unknown " + "or missing API exits 1 with reason unknown_runtime_api.") parser.add_argument("--region", required=True, help="AWS region") parser.add_argument("--account-id", required=True, help="AWS account ID") parser.add_argument("--output", help="Output file (default: stdout)") parser.add_argument("--runtime-api", choices=sorted(RUNTIME_APIS), - help="Selected runtime API (bedrock_models[].required_client_api); " - "responses/chat_completions add the default-project grant for " - "runtime GPT CRIS ids, converse/messages do not. Omit for the " - "id-only behaviour.") + help="Selected runtime API applied to every --models id " + "(ignored with --target); responses/chat_completions add the " + "default-project grant for runtime GPT CRIS ids, converse/messages " + "do not. Omit for the id-only behaviour.") parser.add_argument("--migration-path", help="Plan's code_migration.migration_path; an unsupported lane " "(mantle_messages) exits 1 with a JSON reason instead of a policy.") args = parser.parse_args() - model_ids = [m.strip() for m in args.models.split(",") if m.strip()] + def _fail(reason: str, detail: str) -> None: + print(json.dumps({"ok": False, "reason": reason, "lane": args.migration_path, + "detail": detail}, indent=2)) + sys.exit(1) + + if args.migration_path in UNSUPPORTED_LANES: + # The lane stop wins over every other check: nothing on this path is authorized. + try: + generate_policy([], args.region, args.account_id, migration_path=args.migration_path) + except ValueError as e: + _fail("unsupported_lane", str(e)) + target_apis = None + if args.target: + try: + targets = parse_targets(args.target) + except ValueError as e: + _fail("unknown_runtime_api", str(e)) + model_ids = [m for m, _ in targets] + target_apis = dict(targets) + else: + model_ids = [m.strip() for m in args.models.split(",") if m.strip()] try: policy = generate_policy(model_ids, args.region, args.account_id, - runtime_api=args.runtime_api, - migration_path=args.migration_path) + runtime_api=args.runtime_api if not args.target else None, + migration_path=args.migration_path, target_apis=target_apis) except ValueError as e: - print(json.dumps({"ok": False, "reason": "unsupported_lane" - if args.migration_path in UNSUPPORTED_LANES else "unknown_runtime_api", - "lane": args.migration_path, "detail": str(e)}, indent=2)) - sys.exit(1) + _fail("unknown_runtime_api", str(e)) output = json.dumps(policy, indent=2) + "\n" if args.output: diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py index e93014ed4..8e0a557f5 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py @@ -18,6 +18,7 @@ mantle_project_arn, inference_profile_arn, is_inference_profile, + parse_targets, requires_default_project_grant, ) @@ -542,3 +543,111 @@ def test_cli_mantle_messages_exits_with_structured_reason(self): assert out["lane"] == "mantle_messages" assert "workspaces" in out["detail"] assert "bedrock:InvokeModel" not in result.stdout.replace(out["detail"], "") + + +class TestPerRowTargetApis: + """PR 585 review round 1, B1: the §6.4 recipe used to pass the primary's single + `--runtime-api` for every id, so a runtime_messages plan with a GPT Chat/Responses + secondary lost that secondary's default-project grant. `--target ID=API` carries + each row's own API, exactly like preflight.""" + + CLAUDE = "global.anthropic.claude-sonnet-5" + GPT = "us.openai.gpt-6-astra" + NOVA = "us.amazon.nova-lite-v1:0" + EMBED = "amazon.titan-embed-text-v2:0" + + def _invoke_resources(self, pol): + return {s["Sid"]: s for s in pol["Statement"]}["BedrockInvokeModelScoped"]["Resource"] + + def test_plan_wide_messages_api_is_the_regression_the_per_row_form_fixes(self): + # The reviewer's spot-check: one plan-wide `messages` drops the GPT secondary's grant. + pol = generate_policy([self.CLAUDE, self.GPT], REGION, ACCOUNT, + runtime_api="messages", migration_path="runtime_messages") + assert default_project_arn(REGION, ACCOUNT) not in self._invoke_resources(pol) + + def test_claude_messages_plus_gpt_chat_grants_project_default_exactly_once(self): + pol = generate_policy([self.CLAUDE, self.GPT], REGION, ACCOUNT, + migration_path="runtime_messages", + target_apis={self.CLAUDE: "messages", self.GPT: "chat_completions"}) + resources = self._invoke_resources(pol) + assert resources.count(default_project_arn(REGION, ACCOUNT)) == 1 + assert "arn:aws:bedrock:*::foundation-model/anthropic.claude-sonnet-5" in resources + assert inference_profile_arn(self.GPT, REGION, ACCOUNT) in resources + assert [s["Sid"] for s in pol["Statement"]] == ["BedrockInvokeModelScoped", "BedrockGetInferenceProfile"] + + @pytest.mark.parametrize("gpt_api", ["responses", "chat_completions"]) + def test_gpt_secondary_grant_follows_its_own_row_api(self, gpt_api): + pol = generate_policy([self.CLAUDE, "in.openai.gpt-5.6-terra"], REGION, ACCOUNT, + target_apis={self.CLAUDE: "messages", "in.openai.gpt-5.6-terra": gpt_api}) + assert default_project_arn(REGION, ACCOUNT) in self._invoke_resources(pol) + + def test_claude_messages_plus_nova_converse_plus_embeddings_emits_no_project_default(self): + pol = generate_policy([self.CLAUDE, self.NOVA, self.EMBED], REGION, ACCOUNT, + migration_path="runtime_messages", + target_apis={self.CLAUDE: "messages", self.NOVA: "converse", self.EMBED: "invoke"}) + resources = self._invoke_resources(pol) + assert default_project_arn(REGION, ACCOUNT) not in resources + assert f"arn:aws:bedrock:*::foundation-model/{self.EMBED}" in resources + + def test_gpt_row_on_converse_beside_a_gpt_row_on_chat_grants_once(self): + # Per-row decision, not per-family: the same policy call grants the Chat row and + # withholds for the Converse row; one project ARN results. + chat, conv = "us.openai.gpt-5.4", "global.openai.gpt-5.5" + pol = generate_policy([chat, conv], REGION, ACCOUNT, + target_apis={chat: "chat_completions", conv: "converse"}) + assert self._invoke_resources(pol).count(default_project_arn(REGION, ACCOUNT)) == 1 + assert requires_default_project_grant(chat, "chat_completions") + assert not requires_default_project_grant(conv, "converse") + + def test_row_api_overrides_plan_wide_runtime_api(self): + pol = generate_policy([self.CLAUDE, self.GPT], REGION, ACCOUNT, runtime_api="messages", + target_apis={self.GPT: "chat_completions"}) + assert default_project_arn(REGION, ACCOUNT) in self._invoke_resources(pol) + + def test_unknown_row_api_fails_closed(self): + with pytest.raises(ValueError, match="Unknown runtime API 'grpc'"): + generate_policy([self.GPT], REGION, ACCOUNT, target_apis={self.GPT: "grpc"}) + + def test_parse_targets_builds_pairs_from_repeated_and_comma_joined_specs(self): + assert parse_targets([f"{self.CLAUDE}=messages", f"{self.GPT}=chat_completions,{self.EMBED}=invoke"]) == [ + (self.CLAUDE, "messages"), (self.GPT, "chat_completions"), (self.EMBED, "invoke"), + ] + + @pytest.mark.parametrize("spec", ["us.openai.gpt-6-astra", "us.openai.gpt-6-astra=", "us.openai.gpt-6-astra=grpc"]) + def test_parse_targets_fails_closed_on_missing_or_unknown_api(self, spec): + with pytest.raises(ValueError, match="Every --target must be"): + parse_targets([spec]) + + def test_cli_target_pairs_grant_the_gpt_secondary_under_a_messages_primary(self): + result = subprocess.run( # nosec B603 + [sys.executable, str(Path(__file__).parent / "iam_policy.py"), + "--target", f"{self.CLAUDE}=messages", "--target", f"{self.GPT}=chat_completions", + "--region", REGION, "--account-id", ACCOUNT, "--migration-path", "runtime_messages"], + capture_output=True, text=True, + ) + assert result.returncode == 0, result.stderr + policy = json.loads(result.stdout) + assert policy["Statement"][0]["Resource"].count(default_project_arn(REGION, ACCOUNT)) == 1 + + def test_cli_target_with_unknown_api_exits_with_unknown_runtime_api(self): + result = subprocess.run( # nosec B603 + [sys.executable, str(Path(__file__).parent / "iam_policy.py"), + "--target", f"{self.CLAUDE}=messages", "--target", self.GPT, + "--region", REGION, "--account-id", ACCOUNT], + capture_output=True, text=True, + ) + assert result.returncode == 1 + out = json.loads(result.stdout) + assert out["ok"] is False and out["reason"] == "unknown_runtime_api" + assert "" in out["detail"] + assert "Statement" not in result.stdout + + def test_cli_models_and_target_are_mutually_exclusive(self): + result = subprocess.run( # nosec B603 + [sys.executable, str(Path(__file__).parent / "iam_policy.py"), + "--models", self.GPT, "--target", f"{self.GPT}=responses", + "--region", REGION, "--account-id", ACCOUNT], + capture_output=True, text=True, + ) + assert result.returncode == 2 + assert "not allowed with" in result.stderr diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md index f9c7b04a4..8e874d3eb 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md @@ -341,8 +341,9 @@ limited to: a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, "Permissions", read 2026-11-07), and each model card repeats the requirement for its runtime CRIS ids. - `iam_policy.py --runtime-api ` emits it for `responses` / `chat_completions` rows and - omits it for a `converse` row and for every non-OpenAI id + `iam_policy.py --target =` (one pair per `bedrock_models[]` row, each + row's own API) emits it for `responses` / `chat_completions` rows and omits it for a `converse` or + `messages` row and for every non-OpenAI id — a GPT secondary beside a Claude Messages primary still gets it **Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, diff --git a/plugins/aws-startup-advisor/tests/test_runtime_default_review_round1.py b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round1.py index 2c3019e23..ef7ccfce0 100644 --- a/plugins/aws-startup-advisor/tests/test_runtime_default_review_round1.py +++ b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round1.py @@ -87,6 +87,41 @@ def test_setup_recipes_grant_project_default_to_every_runtime_openai_target() -> assert "GPT-6 / GPT-5.5 / GPT-5.4\n also need" not in text, path +def test_report_generator_iam_recipe_passes_one_target_pair_per_row() -> None: + """Round-1 review B1: the §6.4 recipe must not apply the primary's single + `Target API surface` to every id — a GPT Chat/Responses secondary under a + Claude Messages primary would lose its default-project grant.""" + text = REPORT_GENERATOR.read_text() + raw = _section(text, "# 6.4 Generate scoped IAM policy artifact", "# 7. Generate report") + code = _section(raw, "```bash", "```\n") + recipe = " ".join(raw.split()) + assert "--target =" in code + assert "one per bedrock_models[] row" in code + assert "--runtime-api" not in code, "the recipe command must not pass one plan-wide API" + assert "--migration-path " in code + assert "ai_architecture.bedrock_models[]" in recipe and "aws-design-ai.json" in recipe + assert "Do NOT apply the primary's surface to the other rows" in recipe + assert "unknown_runtime_api" in recipe and "do not substitute an API" in recipe + assert "`invoke` for an embedding row" in recipe + # The legacy single-API form is documented as single-row only, never as the default. + assert "use it only for a single-row plan" in recipe + for path in (CANONICAL_GUIDE, GCP_GENERATE_AI, GCP_GENERATE_ARTIFACTS, AZURE_GENERATE_ARTIFACTS): + assert "iam_policy.py --target =" in path.read_text(), path + assert "iam_policy.py --runtime-api" not in path.read_text(), path + + +def test_iam_helper_grants_the_gpt_secondary_under_a_messages_primary_per_row() -> None: + iam = _load("iam_policy") + claude, gpt, nova = "global.anthropic.claude-sonnet-5", "us.openai.gpt-6-astra", "us.amazon.nova-lite-v1:0" + project = iam.default_project_arn("us-east-1", "123456789012") + mixed = iam.generate_policy([claude, gpt], "us-east-1", "123456789012", migration_path="runtime_messages", + target_apis=dict(iam.parse_targets([f"{claude}=messages", f"{gpt}=chat_completions"]))) + assert mixed["Statement"][0]["Resource"].count(project) == 1 + converse = iam.generate_policy([claude, nova], "us-east-1", "123456789012", migration_path="runtime_messages", + target_apis={claude: "messages", nova: "converse"}) + assert project not in converse["Statement"][0]["Resource"] + + def test_vendored_openai_guides_match_canonical() -> None: for vendored in VENDORED_GUIDES: assert vendored.read_bytes() == CANONICAL_GUIDE.read_bytes(), vendored From 1cb59c88da68a6e418ad3581f6ae3920007aa131 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 7 Nov 2026 20:25:49 -0800 Subject: [PATCH 14/41] fix(aws-startup-advisor): grant the default project any-row-wins when one Bedrock id sits in several --target rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR 585 review round 1, pass 2, blocker B2: `target_apis = dict(targets)` kept only the LAST row's API per id, so `--target us.openai.gpt-5.6-sol=chat_completions --target us.openai.gpt-5.6-sol=converse` emitted 0 project/default while the reverse order emitted 1. The §6.4 recipe emits one pair per bedrock_models[] row without deduplicating, and the schema lets a GPT CRIS id appear on a converse row (Guardrails / application-inference-profile call site) and a chat_completions/responses row for a second source model. - iam_policy.py: new pure `target_apis_by_id(pairs) -> dict[str, set[str]]` groups every API an id is called on; `generate_policy(target_apis=)` accepts a str or a set per id, validates each API in the set (fail closed on an unknown one), and grants `project/default` once when ANY of the id's rows is responses/chat_completions. CLI `--target` now builds the set instead of `dict(targets)`. - test_iam_policy.py::TestPerRowTargetApis: [gpt=chat,gpt=converse] and the reverse order and [gpt=converse,gpt=responses] all grant exactly once; [gpt=converse, gpt=converse] grants none; target_apis_by_id keeps every row's API; an unknown API inside a row set fails closed; CLI duplicate-id rows grant once in both orders (the reviewer's spot-check). - llm2bedrock-report-generator.md §6.4: tell the agent not to deduplicate rows and that a duplicated id is granted once if any row is responses/chat_completions. Validation (run in this worktree, 2026-11-07): - mise -C plugins/aws-startup-advisor run ci: exit 0 — [test] 1431 passed, 222 subtests passed; [test:telemetry] 311 passed; lint:md 0 errors; vendored-parity PASS 64; artifacts PASS 99; skill-refs PASS 1521. (A stale gitignored skills/llm-to-bedrock/scripts/.venv from the previous step broke lint:md on the first run; it was removed, not committed.) - pytest skills/llm-to-bedrock (uv project env): 622 passed (was 614; +8 new). - pytest skills/agent-advisor: 570 passed (no agent-advisor files touched). - pytest tests/test_gpt54_55_cris_prefix_scope.py: 3 passed. - python3 tools/validate.py: All validations passed. - tools/sync-vendored.py --check: PASS, 64 byte-identical. - git diff --check: clean. - mise run build: exit 1 ONLY on gitleaks' pre-existing upstream finding at f8d09fd1 (tests/test_bedrock_migration_assessment_discover.py:118, generic-api-key); not introduced by this branch. - CLI spot-check: sol=chat+converse -> 1 project/default; converse+chat -> 1; converse+converse -> 0. --- .../agents/llm2bedrock-report-generator.md | 6 ++- .../llm-to-bedrock/scripts/iam_policy.py | 46 ++++++++++++++----- .../llm-to-bedrock/scripts/test_iam_policy.py | 46 +++++++++++++++++++ 3 files changed, 85 insertions(+), 13 deletions(-) diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md index 210f47447..7c0bd4825 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md @@ -578,7 +578,11 @@ uv run --project python /iam_policy.py \ row's API is the `Target API surface` from your context (the value B4 resolved; use it when that row records no `required_client_api`). Do NOT apply the primary's surface to the other rows: the grant is decided per row, so a `runtime_messages` plan whose secondary is a GPT Chat / Responses CRIS id still gets `project/default` -for that secondary, while a Nova Converse secondary does not. If any row's API is missing or unknown the +for that secondary, while a Nova Converse secondary does not. Do not deduplicate rows: when one id appears in +two rows with different APIs (a `converse` row for a Guardrails / application-inference-profile call site and +a `chat_completions` / `responses` row for another source model), pass both pairs in any order — the helper +grants `project/default` once if ANY of that id's rows is on `responses` / `chat_completions`, and never if +none is. If any row's API is missing or unknown the helper exits 1 with `reason: unknown_runtime_api` and writes no policy — report that verbatim, do not substitute an API and do not hand-write a policy. The legacy single-API form `--models --runtime-api ` is still accepted but applies ONE API to diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py index 091a14f3f..52ded4128 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py @@ -186,18 +186,35 @@ def parse_targets(specs: list[str]) -> list[tuple[str, str]]: return pairs +def target_apis_by_id(pairs: list[tuple[str, str]]) -> dict[str, set[str]]: + """Pure: group parsed `(model_id, api)` pairs into EVERY API each id is called on. + + One Bedrock id may sit in two `bedrock_models[]` rows with different APIs — a + `converse` row for a Guardrails / application-inference-profile call site and a + `chat_completions` / `responses` row for a second source model mapped to the same + target. `dict(pairs)` would keep only the last row, so the grant would depend on + row order; the set keeps every row and the grant is any-row-wins.""" + by_id: dict[str, set[str]] = {} + for model_id, api in pairs: + by_id.setdefault(model_id, set()).add(api) + return by_id + + def generate_policy(model_ids: list[str], region: str, account_id: str, runtime_api: str | None = None, migration_path: str | None = None, - target_apis: dict[str, str] | None = None) -> dict: + target_apis: dict[str, str | set[str]] | None = None) -> dict: """Build a scoped IAM policy covering exactly the given model IDs. `runtime_api` is the plan's selected API (`responses`, `chat_completions`, `converse`, `messages`); it decides the default-project grant (see `requires_default_project_grant`). None keeps the id-only behaviour. - `target_apis` maps a model id to ITS OWN row's API and takes precedence over - `runtime_api` for that id, so a mixed plan (Claude Messages primary + GPT Chat - secondary) grants the default project to the GPT row and not to the Claude row. + `target_apis` maps a model id to ITS OWN row's API — a single API, or the set + of APIs from every row that names the id (see `target_apis_by_id`) — and takes + precedence over `runtime_api` for that id, so a mixed plan (Claude Messages + primary + GPT Chat secondary) grants the default project to the GPT row and not + to the Claude row. When one id carries several APIs, ANY row on `responses` / + `chat_completions` grants it, regardless of row order. `migration_path` is the plan's `code_migration.migration_path`; an UNSUPPORTED lane (`mantle_messages`) raises `ValueError` instead of emitting runtime `bedrock:InvokeModel*` for a bare Claude id that the lane would never @@ -228,11 +245,13 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, if runtime_api is not None and runtime_api not in RUNTIME_APIS: raise ValueError(f"Unknown runtime API {runtime_api!r}; expected one of " f"{sorted(RUNTIME_APIS)}") - target_apis = dict(target_apis or {}) - for mid, api in target_apis.items(): - if api not in TARGET_APIS: - raise ValueError(f"Unknown runtime API {api!r} for target {mid!r}; expected one of " - f"{sorted(TARGET_APIS)}") + target_apis = {mid: {apis} if isinstance(apis, str) else set(apis) + for mid, apis in (target_apis or {}).items()} + for mid, apis in target_apis.items(): + for api in sorted(apis): + if api not in TARGET_APIS: + raise ValueError(f"Unknown runtime API {api!r} for target {mid!r}; expected one of " + f"{sorted(TARGET_APIS)}") runtime_ids = [m for m in model_ids if not is_mantle_model(m, region)] mantle_ids = [m for m in model_ids if is_mantle_model(m, region)] @@ -249,8 +268,11 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, # InvokeModel on the inference-profile ARN AND the default project # ARN; both are required, neither replaces the other. Converse-only # routes, Anthropic ids and every other runtime target keep the - # 2-ARN shape above. The row's own API wins over the plan-wide one. - if requires_default_project_grant(mid, target_apis.get(mid, runtime_api)): + # 2-ARN shape above. The row's own API wins over the plan-wide one, + # and when the id sits in several rows ANY Responses / Chat row grants + # it (never last-row-wins, so row order cannot drop the grant). + row_apis = target_apis.get(mid) or {runtime_api} + if any(requires_default_project_grant(mid, api) for api in row_apis): resources.append(default_project_arn(region, account_id)) else: resources.append(foundation_model_arn(mid, region)) @@ -347,7 +369,7 @@ def _fail(reason: str, detail: str) -> None: except ValueError as e: _fail("unknown_runtime_api", str(e)) model_ids = [m for m, _ in targets] - target_apis = dict(targets) + target_apis = target_apis_by_id(targets) else: model_ids = [m.strip() for m in args.models.split(",") if m.strip()] try: diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py index 8e0a557f5..4521e397b 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy.py @@ -20,6 +20,7 @@ is_inference_profile, parse_targets, requires_default_project_grant, + target_apis_by_id, ) REGION = "us-east-1" @@ -604,6 +605,51 @@ def test_row_api_overrides_plan_wide_runtime_api(self): target_apis={self.GPT: "chat_completions"}) assert default_project_arn(REGION, ACCOUNT) in self._invoke_resources(pol) + # PR 585 review round 1, B2: the same GPT CRIS id in two rows (a Converse row for a + # Guardrails call site, a Chat row for a second source model). `dict(pairs)` kept + # only the last row, so the grant depended on row order. + SOL = "us.openai.gpt-5.6-sol" + + @pytest.mark.parametrize("specs", [ + [f"{SOL}=chat_completions", f"{SOL}=converse"], + [f"{SOL}=converse", f"{SOL}=chat_completions"], + [f"{SOL}=converse", f"{SOL}=responses"], + ]) + def test_same_id_on_chat_and_converse_rows_grants_once_in_either_order(self, specs): + pairs = parse_targets(specs) + pol = generate_policy([m for m, _ in pairs], REGION, ACCOUNT, + target_apis=target_apis_by_id(pairs)) + assert self._invoke_resources(pol).count(default_project_arn(REGION, ACCOUNT)) == 1 + + def test_same_id_on_two_converse_rows_grants_none(self): + pairs = parse_targets([f"{self.SOL}=converse", f"{self.SOL}=converse"]) + pol = generate_policy([m for m, _ in pairs], REGION, ACCOUNT, + target_apis=target_apis_by_id(pairs)) + assert default_project_arn(REGION, ACCOUNT) not in self._invoke_resources(pol) + + def test_target_apis_by_id_keeps_every_row_api(self): + assert target_apis_by_id([(self.SOL, "chat_completions"), (self.SOL, "converse"), + (self.CLAUDE, "messages")]) == { + self.SOL: {"chat_completions", "converse"}, self.CLAUDE: {"messages"}, + } + + def test_unknown_api_inside_a_row_set_fails_closed(self): + with pytest.raises(ValueError, match="Unknown runtime API 'grpc'"): + generate_policy([self.SOL], REGION, ACCOUNT, target_apis={self.SOL: {"converse", "grpc"}}) + + @pytest.mark.parametrize("order", [("chat_completions", "converse"), ("converse", "chat_completions")]) + def test_cli_duplicate_id_rows_grant_once_regardless_of_order(self, order): + # The reviewer's spot-check: chat-then-converse used to emit 0 project/default. + result = subprocess.run( # nosec B603 + [sys.executable, str(Path(__file__).parent / "iam_policy.py"), + "--target", f"{self.SOL}={order[0]}", "--target", f"{self.SOL}={order[1]}", + "--region", REGION, "--account-id", ACCOUNT], + capture_output=True, text=True, + ) + assert result.returncode == 0, result.stderr + policy = json.loads(result.stdout) + assert policy["Statement"][0]["Resource"].count(default_project_arn(REGION, ACCOUNT)) == 1 + def test_unknown_row_api_fails_closed(self): with pytest.raises(ValueError, match="Unknown runtime API 'grpc'"): generate_policy([self.GPT], REGION, ACCOUNT, target_apis={self.GPT: "grpc"}) From 8e431e37df86f167ba59857d2562e26bea1000c5 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 08:54:32 -0800 Subject: [PATCH 15/41] =?UTF-8?q?fix(aws-startup-advisor):=20address=20PR?= =?UTF-8?q?=20585=20review=20round=202=20=E2=80=94=20validated=20ids=20in?= =?UTF-8?q?=20the=20=C2=A76.4=20IAM=20recipe,=20project-scoped=20GetInvoke?= =?UTF-8?q?=20for=20retained=20stored-response=20ops,=20identity-checked?= =?UTF-8?q?=20saved-API=20reuse=20in=20B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three P2 findings from leon1418 on f3c01556 (threads 4505069045, 4505069047, 4505069050). 1. Report §6.4 IAM recipe reloaded aws-design-ai.json's ORIGINAL ids, so a C4 target change (Sol → Terra, recorded in resolved_model_overrides) emitted Sol permissions and omitted the Terra profile. The recipe now joins each bedrock_models[] row's required_client_api to the validated analysis target_models (overrides applied) and never falls back to the row's aws_model_id; iam_policy.validated_target_pairs() does the join and fails closed on a row without a validated entry or an API. Regression: a C4 Sol → Terra change yields a C6 policy naming Terra (+ project/default) and no Sol resource; the pre-fix design-row path is kept as the failing control. 2. Retained client.responses.retrieve(id) / cancel / delete on bedrock-runtime authorize bedrock:GetInvoke / CancelInvoke / DeleteInvoke on project/default, separately from the InvokeModel creation needs (inference-responses-api.html "Permissions", read 2026-11-08). analysis.json gains optional stored_response_ops (enum retrieve|cancel|delete); the analyzer §8.3 scans for the call sites; iam_policy.py gains --stored-response-ops and emits a BedrockStoredResponseOps statement with ONLY the retained actions on project/default, and only when a --target row is a runtime GPT CRIS id on `responses` — Converse, Anthropic Messages, Chat Completions, embeddings, bare mantle ids and id-only legacy calls are refused (reason stored_response_ops_no_responses_row). The report recipe passes the field through, the How to Apply checklist gains a create-plus-retrieve permission check, and the GCP / Azure setup_bedrock contracts, GCP generate-ai and the shared OpenAI guide (vendored copies re-synced) name the grant, its controls and the create-plus-retrieve verification. 3. SKILL.md B4 read the saved resolved_runtime_api before C0's assess_design_sha256 check, so a cached `messages` answer could send a new GPT row to the Anthropic endpoint. B4 now runs preflight_bedrock.py --resume-runtime-api --design , which reuses the saved answer only when it exists AND the saved assess_design_sha256 and target_models (source, aws_model_id) identity match the current design; any mismatch, missing context or missing field forces re-resolution before any probe. The context-block recovery sentence points at the same gate. Validation (worktree fix-llm-to-bedrock-runtime-default): - mise -C plugins/aws-startup-advisor run ci: exit 0 — test 1441 passed (+222 subtests), test:telemetry 311 passed, lint:md 0 errors, vendored-parity PASS (64), skill-refs PASS, model-id OK, artifacts PASS, consent-disclosure OK, model-catalog-drift OK (3 pre-existing warns) - llm-to-bedrock scripts pytest (uv run --project .): 655 passed (622 baseline + 33 new) - agent-advisor scripts pytest: 570 passed (unchanged; no agent-advisor edits) - tests/test_gpt54_55_cris_prefix_scope.py and test_runtime_default_review_round1.py green - python3 tools/validate.py: All validations passed - tools/sync-vendored.py --check: PASS - git diff --check: clean - mise run build: exit 1 ONLY on the pre-existing gitleaks finding at upstream f8d09fd1 (tests/test_bedrock_migration_assessment_discover.py:118, generic-api-key on the string "claude-3.5-sonnet"); lint:manifests passed. Not introduced or touched by this commit. --- .../agents/llm2bedrock-code-analyzer.md | 18 +- .../agents/llm2bedrock-report-generator.md | 77 ++++-- .../phases/generate/generate-artifacts-ai.md | 15 ++ .../vendored/ai/ai-openai-to-bedrock.md | 10 + .../references/phases/generate/generate-ai.md | 2 +- .../phases/generate/generate-artifacts-ai.md | 4 +- .../vendored/ai/ai-openai-to-bedrock.md | 10 + .../skills/llm-to-bedrock/SKILL.md | 29 ++- .../llm-to-bedrock/scripts/iam_policy.py | 144 ++++++++++- .../scripts/preflight_bedrock.py | 91 +++++++ .../scripts/schemas/analysis.json | 8 +- .../scripts/test_iam_policy_round2.py | 190 ++++++++++++++ .../test_preflight_resume_runtime_api.py | 108 ++++++++ .../skills/shared/ai/ai-openai-to-bedrock.md | 10 + .../test_runtime_default_review_round2.py | 241 ++++++++++++++++++ 15 files changed, 923 insertions(+), 34 deletions(-) create mode 100644 plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy_round2.py create mode 100644 plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_resume_runtime_api.py create mode 100644 plugins/aws-startup-advisor/tests/test_runtime_default_review_round2.py diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-code-analyzer.md b/plugins/aws-startup-advisor/agents/llm2bedrock-code-analyzer.md index 127169c96..98550b3d0 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-code-analyzer.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-code-analyzer.md @@ -248,8 +248,24 @@ grep -rn "embedding\|embed_query\|create_embedding" --include="*.py" --in # Vision / image input grep -rn "image_url\|image_file\|vision\|ImageBlock" --include="*.py" --include="*.js" --include="*.ts" | grep -v node_modules + +# Stored-response operations (OpenAI Responses API only): retrieve / cancel / delete by id +grep -rn "responses\.retrieve(\|responses\.cancel(\|responses\.delete(\|/responses/[^\"']*\(/cancel\)\?[\"']" --include="*.py" --include="*.js" --include="*.ts" | grep -v node_modules ``` +**`stored_response_ops`** (typed field, optional — emit it whenever the source is an OpenAI +Responses app): the subset of `["retrieve", "cancel", "delete"]` whose `client.responses.(id)` +call sites (or raw `GET /responses/{id}`, `POST /responses/{id}/cancel`, `DELETE /responses/{id}` +requests) the last grep found and a runtime rewrite keeps. Creation (`responses.create`) is NOT +one of them. On `bedrock-runtime`, retrieving, canceling or deleting a stored response authorizes +`bedrock:GetInvoke` / `bedrock:CancelInvoke` / `bedrock:DeleteInvoke` on the account's default +project, separately from the `bedrock:InvokeModel` that creation needs +(, "Permissions", +read 2026-11-08) — the report generator's §6.4 IAM recipe reads this field so a retained +`responses.retrieve(id)` is not denied after creation and the preflight ping both passed. Emit `[]` +(or omit) when no such call exists; never emit it for a Chat Completions, Converse or Anthropic +Messages source, which have no stored responses. + # 9. Detect behavior deltas (user-visible parameter-surface differences) If `source_provider ∈ {openai, google, anthropic}` AND `same_model_family == false`, scan the source code for known parameter-surface differences between the source provider and Bedrock. The orchestration checkpoint asks the user to confirm each user-visible change before the rewriter modifies code; this step enumerates them. (Vertex AI customers are emitted as `google` per §7.1.2 — parameter surface is identical between Gemini API and Vertex AI Gemini.) @@ -365,7 +381,7 @@ Write your result to `/analysis.json` with the `Write` Return ONE flat object: the typed fields and `summary` are all top-level siblings (no `data` wrapper — the strict schema rejects a nested `data` key). -- **Typed fields** — the fields in the analysis schema (`AiAnalysisData`), at top level. Always populate every required field; use `""` / `0` / `[]` / `false` / `"none"` for absent values. The nested `special_patterns` object MUST include all four booleans. +- **Typed fields** — the fields in the analysis schema (`AiAnalysisData`), at top level. Always populate every required field; use `""` / `0` / `[]` / `false` / `"none"` for absent values. The nested `special_patterns` object MUST include all four booleans. `stored_response_ops` (§8.3) is the one optional typed field: an array drawn from `["retrieve", "cancel", "delete"]`, `[]` or omitted when the source retains no stored-response operation. - **`summary`** — short prose for the user / sidebar, a top-level field alongside the typed fields. ~1–3 sentences. Mention framework, file count, key special patterns. - **`errors`** — string log of resolution decisions and warnings: rule-of-thumb / defaulted entries from §7.2 Tier 2/3, model-ID corrections from §10, the Vertex substring from §7.1.2, and any non-fatal scan failures. **Multiple entries: join with `"; "` (semicolon + space) on a single line.** Use `"none"` if nothing notable. diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md index 7c0bd4825..eb902e617 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md @@ -34,7 +34,7 @@ Read accumulated state from prompt context (forwarded from every prior Track 2 p - **``** — AWS region for Bedrock (the `AWS region:` line in your context). - **AWS profile** — the `AWS profile` line, when present: prepend `AWS_PROFILE=` inline to the §6.1 pricing-script invocation and any other aws/boto3 command; omit when absent. - **``** — the pinned-toolchain scripts directory, the `Scripts directory (pinned uv toolchain):` line in your context. Used to run `bedrock_pricing.py`. -- **From `llm2bedrock-code-analyzer` (`AiAnalysisData`)** — `source_provider`, `ai_framework`, `source_models`, `target_models` (` -> ` pairs), `coverage_level`, `use_case_type`, `errors`. +- **From `llm2bedrock-code-analyzer` (`AiAnalysisData`)** — `source_provider`, `ai_framework`, `source_models`, `target_models` (` -> ` pairs — the FINAL validated ids §6.4's IAM recipe joins the plan rows to), `stored_response_ops` (optional: retained `responses.retrieve` / `cancel` / `delete` ops, §6.4), `coverage_level`, `use_case_type`, `errors`. - **From `llm2bedrock-log-ingestor` (`LogIngestionData`)** — `total_golden_cases`, `coverage_level`, `gaps`. Drives the Risk Assessment + Coverage sections. - **From `llm2bedrock-prompt-evaluator` (`EvalData` + `notes` prefixes)** — top-level fields: - `pass_rate` (fraction in [0,1]), `total_cases`, `failures`, `live_source_baseline` (bool), `judge_model`. @@ -566,29 +566,62 @@ Generate a least-privilege IAM policy scoped to the exact model ARNs selected du ```bash uv run --project python /iam_policy.py \ --target = [--target ... one per bedrock_models[] row] \ + [--stored-response-ops ] \ --region \ --account-id \ --migration-path \ --output /.saws-migrate/iam-policy.json ``` -**One `--target` pair per row, each with ITS OWN API.** Read `ai_architecture.bedrock_models[]` from -`/aws-design-ai.json` and emit `=` for every row -(`responses` | `chat_completions` | `converse` | `messages`; `invoke` for an embedding row). The primary chat -row's API is the `Target API surface` from your context (the value B4 resolved; use it when that row records -no `required_client_api`). Do NOT apply the primary's surface to the other rows: the grant is decided per row, -so a `runtime_messages` plan whose secondary is a GPT Chat / Responses CRIS id still gets `project/default` -for that secondary, while a Nova Converse secondary does not. Do not deduplicate rows: when one id appears in -two rows with different APIs (a `converse` row for a Guardrails / application-inference-profile call site and -a `chat_completions` / `responses` row for another source model), pass both pairs in any order — the helper -grants `project/default` once if ANY of that id's rows is on `responses` / `chat_completions`, and never if -none is. If any row's API is missing or unknown the -helper exits 1 with `reason: unknown_runtime_api` and writes no policy — report that verbatim, do not -substitute an API and do not hand-write a policy. The legacy single-API form -`--models --runtime-api ` is still accepted but applies ONE API to -every id; use it only for a single-row plan. A `mantle_messages` path never reaches this agent (the -orchestrator's lane gate stops it before Phase C); if it somehow does, the helper exits 1 with -`reason: unsupported_lane` — report that verbatim and do not hand-write a policy. +**The ids come from the VALIDATED mapping, the APIs from the plan rows — join them.** `` in the +command above is the VALIDATED target id for that row, not the design row's own `aws_model_id`. +`aws-design-ai.json` keeps the ORIGINAL design id after the user changes a target in C4 (recorded in run-context +`resolved_model_overrides`; the dispatch context's `Target Bedrock model(s)` line and the re-run +analysis already carry the new target), so re-reading `bedrock_models[].aws_model_id` would emit the old +model's permissions — a Sol → Terra change would grant `us.openai.gpt-5.6-sol` and omit the Terra profile +the rewritten app calls. Instead, for every `ai_architecture.bedrock_models[]` row in +`/aws-design-ai.json`, take the row's `source_model_id` (older plans `source_model`), +look it up in the analyzer's validated `target_models` (`" -> "`, one entry per +row — the right-hand side is the FINAL validated id with every override applied), and emit +`=`. This is exactly `iam_policy.validated_target_pairs(rows, +analysis["target_models"], primary_model_id, primary_api)` in ``; use it when a row's API has to +be filled in, and run `python -c` against it rather than hand-joining. Control: the set of ids you emit MUST +equal the set on your context's `Target Bedrock model(s)` line; a row whose source model has no validated +entry makes the helper raise — stop and report it, never fall back to the row's original `aws_model_id`. + +**One `--target` pair per row, each with ITS OWN API** (`responses` | `chat_completions` | `converse` | +`messages`; `invoke` for an embedding row). The primary chat row's API is the `Target API surface` from +your context (the value B4 resolved; use it when that row records no `required_client_api`). Do NOT apply +the primary's surface to the other rows: the grant is decided per row, so a `runtime_messages` plan whose +secondary is a GPT Chat / Responses CRIS id still gets `project/default` for that secondary, while a Nova +Converse secondary does not. Do not deduplicate rows: when one id appears in two rows with different APIs (a +`converse` row for a Guardrails / application-inference-profile call site and a `chat_completions` / +`responses` row for another source model), pass both pairs in any order — the helper grants +`project/default` once if ANY of that id's rows is on `responses` / `chat_completions`, and never if none +is. If any row's API is missing or unknown the helper exits 1 with `reason: unknown_runtime_api` and writes +no policy — report that verbatim, do not substitute an API and do not hand-write a policy. The legacy +single-API form `--models --runtime-api ` is still accepted but +applies ONE API to every id; use it only for a single-row plan (and still with the validated id, not the +design's). A `mantle_messages` path never reaches this agent (the orchestrator's lane gate stops it before +Phase C); if it somehow does, the helper exits 1 with `reason: unsupported_lane` — report that verbatim and +do not hand-write a policy. + +**Retained stored-response operations need their own grant.** Read `stored_response_ops` from +`analysis.json` (the analyzer's §8.3 scan for `client.responses.retrieve(id)` / `.cancel(id)` / +`.delete(id)` call sites the rewrite keeps). When it is non-empty, pass +`--stored-response-ops ` and the helper adds a `BedrockStoredResponseOps` statement +on `arn:aws:bedrock:::project/default` with ONLY the retained actions — +`bedrock:GetInvoke` for `retrieve`, `bedrock:CancelInvoke` for `cancel`, `bedrock:DeleteInvoke` for +`delete`: creating a response authorizes `bedrock:InvokeModel` on the inference target and the default +project, while retrieving, canceling or deleting a stored response authorizes those three actions on the +project (, +"Permissions", read 2026-11-08). Without it, creation and the preflight ping pass while the app's +existing `responses.retrieve(id)` is denied. Stored responses exist only on the runtime Responses API: the +helper emits the statement only when a `--target` row is a runtime GPT CRIS id on `responses`, and exits 1 +with `reason: stored_response_ops_no_responses_row` for a Converse / Anthropic Messages / Chat Completions +plan — report that verbatim (the analyzer should not have recorded ops for such a source) and do not add the +actions by hand. When the field is absent or `[]`, omit the flag; the policy then carries no `GetInvoke` / +`CancelInvoke` / `DeleteInvoke`. The script handles the dual-ARN pattern: foundation-model ARNs for plain model IDs and inference-profile ARNs for geo-prefixed IDs (e.g. `us.anthropic.claude-sonnet-4-6`). For any cross-region (CRIS) target it also emits a `BedrockGetInferenceProfile` statement (`bedrock:GetInferenceProfile` on the profile ARNs) — the documented prerequisite for invoking through a profile (, read 2026-11-07). For every runtime Responses / Chat Completions target — any proprietary GPT CRIS id by id form (GPT-5.6 Sol / Terra / Luna, the GPT-6 family, GPT-5.5 / GPT-5.4: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`, or their `inference-profile/` ARNs) whose own `--target` API is `responses` or `chat_completions` (legacy form: `--runtime-api` set to one of those, or omitted) — it adds a third resource, `arn:aws:bedrock:::project/default`, because a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, read 2026-11-07) and each of those model cards (read 2026-11-07) repeats the requirement; do not strip it from the report's policy excerpt. A row on `converse` or `messages` and every Anthropic / Nova / gpt-oss / embedding id get no project resource. The output is a ready-to-use IAM policy JSON file. @@ -883,6 +916,14 @@ line `*Monthly estimate unavailable — sample has no time-span information.*` - `.env.example` sets `AWS_REGION` and the model id. Access keys stay commented. A Mantle bearer token, when present, is commented, expires within 12 hours, and is for a short local run — not an ECS task or a Lambda environment. - Apply the generated least-privilege IAM policy: `.saws-migrate/iam-policy.json` (scoped to the exact model ARNs used in this migration — review before attaching to a role) + - + **Create-plus-retrieve permission check** (the preflight ping only exercises creation): under + the attached role, run `r = client.responses.create(model="", input="ping")` + and then `client.responses.retrieve(r.id)` (). A 403 on the second call means the policy's `BedrockStoredResponseOps` + statement (`bedrock:GetInvoke` <, `CancelInvoke`, `DeleteInvoke`> on + `arn:aws:bedrock:::project/default`) is not attached — creation passing does not + prove retrieval is authorized. 3. **Run tests:** diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md index feb86e8d5..3823bea19 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -106,6 +106,21 @@ every Anthropic / Nova / gpt-oss id. The mantle `bedrock-mantle:*` policy (bare `openai.gpt-5.5` / `openai.gpt-5.4` / `openai.gpt-5.6-*` / `openai.gpt-6-*` ids included) is unaffected. +**Retained stored-response operations.** When the source code keeps `client.responses.retrieve(id)` +(or `.cancel(id)` / `.delete(id)`; scan for `responses.retrieve(` / `responses.cancel(` / +`responses.delete(`), a runtime `responses` row ALSO needs `bedrock:GetInvoke` (and +`bedrock:CancelInvoke` / `bedrock:DeleteInvoke` only for the ops actually retained) on the same +`arn:aws:bedrock:{region}:{account}:project/default`: creating a response authorizes +`bedrock:InvokeModel` on the target and the project, while retrieving, canceling or deleting a +stored response authorizes those actions on the project (same page, "Permissions", read +2026-11-08) — without them creation and the ping pass and the app's existing retrieve is denied. +`iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits that `BedrockStoredResponseOps` +statement for a runtime `responses` row only and refuses (`reason: +stored_response_ops_no_responses_row`) for a `converse`, `messages` or `chat_completions` plan, +which has no stored responses. `setup_bedrock.sh`'s verification then also retrieves the response it +just created (`r = client.responses.create(...)`, then `client.responses.retrieve(r.id)`): a 403 on +the second call means the grant is missing even though creation passed. + ## Step 3B: `gateway_config.{yaml,py,json}` (gateway users) Format by framework: `llm_router` → `gateway_config.yaml` (LiteLLM); `framework` → diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index 8e874d3eb..11af0d657 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -344,6 +344,16 @@ limited to: `iam_policy.py --target =` (one pair per `bedrock_models[]` row, each row's own API) emits it for `responses` / `chat_completions` rows and omits it for a `converse` or `messages` row and for every non-OpenAI id — a GPT secondary beside a Claude Messages primary still gets it +5. Retained stored-response operations → a source that keeps `client.responses.retrieve(id)` (or `.cancel(id)` / + `.delete(id)`) ALSO needs `bedrock:GetInvoke` (`bedrock:CancelInvoke` / `bedrock:DeleteInvoke`, only for the ops + actually retained) on the same `project/default` ARN: creating a response authorizes `bedrock:InvokeModel` on + the target and the project, while retrieving, canceling or deleting a stored response authorizes those actions + on the project (same page, "Permissions", read 2026-11-08) — without them creation and the preflight ping pass + and the app's existing retrieve is denied. `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits + the `BedrockStoredResponseOps` statement for a runtime `responses` row only and refuses for a `converse` / + `messages` / `chat_completions` plan (`reason: stored_response_ops_no_responses_row`), which has no stored + responses. Verify with create-plus-retrieve: `r = client.responses.create(...)` then + `client.responses.retrieve(r.id)` under the attached role. **Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md index 4fd7ebe74..0b51bb086 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md @@ -30,7 +30,7 @@ Check `preferences.json` → `ai_constraints.ai_framework` to determine the migr **OpenAI SDK users keeping the same model (config path for a Responses source; small reshape for a Chat Completions source)** — `ai_framework` = `direct` AND `ai_source` = `openai` AND (`migration_path` == `"runtime_openai_cris"` (the default) OR `migration_path` **starts with** `mantle` (`mantle`, `mantle_openai_responses`, `mantle_openai_chat`, selector-gated)): -- `runtime_openai_cris` (the default — GPT-5.6/GPT-6/GPT-5.5/GPT-5.4 via `bedrock-runtime` CRIS): point the client at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` — **not** the mantle URL. Update the model string to the CRIS inference-profile ID the selected model's card lists (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), not the bare ID. Grant standard `bedrock:InvokeModel*` actions against the CRIS inference-profile ARN — **not** `bedrock-mantle:*`. For every runtime Responses / Chat Completions target — any proprietary GPT CRIS id by id form: GPT-5.6 Sol / Terra / Luna, the GPT-6 family, and GPT-5.5 / GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` alongside the inference-profile ARN: the Responses API authorizes the default project separately from the inference target (, read 2026-11-07) and each model card repeats it for its runtime CRIS ids; `iam_policy.py --target =` (one pair per row, each row's own API) emits this third ARN for exactly the `responses` / `chat_completions` rows — a GPT secondary beside a Claude Messages primary included — and omits it for a `converse` or `messages` row. Read `aws-design-ai.json` → `ai_architecture.bedrock_models[].required_client_api` — the API the source app already calls — and emit that surface (`responses.create`, `chat.completions.create`, or Converse when Design recorded `converse` for a Guardrails or application-inference-profile need); for a region-only move it is simply the source's own API. +- `runtime_openai_cris` (the default — GPT-5.6/GPT-6/GPT-5.5/GPT-5.4 via `bedrock-runtime` CRIS): point the client at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` — **not** the mantle URL. Update the model string to the CRIS inference-profile ID the selected model's card lists (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), not the bare ID. Grant standard `bedrock:InvokeModel*` actions against the CRIS inference-profile ARN — **not** `bedrock-mantle:*`. For every runtime Responses / Chat Completions target — any proprietary GPT CRIS id by id form: GPT-5.6 Sol / Terra / Luna, the GPT-6 family, and GPT-5.5 / GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` alongside the inference-profile ARN: the Responses API authorizes the default project separately from the inference target (, read 2026-11-07) and each model card repeats it for its runtime CRIS ids; `iam_policy.py --target =` (one pair per row, each row's own API) emits this third ARN for exactly the `responses` / `chat_completions` rows — a GPT secondary beside a Claude Messages primary included — and omits it for a `converse` or `messages` row. A source that keeps `client.responses.retrieve(id)` (or `.cancel` / `.delete`) also needs `bedrock:GetInvoke` (`CancelInvoke` / `DeleteInvoke`, only the retained ops) on that same `project/default` ARN — managing a stored response authorizes those actions on the project (same page, "Permissions", read 2026-11-08); `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits them for a runtime `responses` row only, never for a `converse` / `messages` / `chat_completions` row. Read `aws-design-ai.json` → `ai_architecture.bedrock_models[].required_client_api` — the API the source app already calls — and emit that surface (`responses.create`, `chat.completions.create`, or Converse when Design recorded `converse` for a Guardrails or application-inference-profile need); for a region-only move it is simply the source's own API. - `mantle` / `mantle_openai_responses` / `mantle_openai_chat` (only when Design recorded a `mantle_selector`, see `references/vendored/ai/bedrock-endpoint-policy.md`): point the client at `https://bedrock-mantle.{region}.api.aws/openai/v1` — note the `openai/v1` segment for proprietary GPT models; a bare `/v1` 404s. Swap the credential for a Bedrock API key or the auto-refreshing token provider — **not** the existing OpenAI key. Update the model string to the bare Bedrock model ID (`openai.gpt-*` for a same-model move). Grant the `bedrock-mantle:*` actions; `bedrock:InvokeModel` does not authorize these models. - **Config-only if the source already calls `responses.create`.** A Chat Completions source for GPT-6 or GPT-5.6 may keep `chat.completions.create`/`choices[0].message` parsing unchanged — Chat Completions is supported on both mantle and runtime CRIS for these models, so reshape to Responses is optional (only for a Responses-only feature or demonstrated constraint). GPT-5.5/GPT-5.4 follow the same rule — their model cards (read 2026-11-06) list Chat Completions on both mantle and runtime, so the reshape is optional for them too; probe the selected API against the target account before committing, as for any model. - Test in staging, validate responses diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md index 2073593ea..d3ee1f9b0 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -215,12 +215,12 @@ Generate `ai-migration/setup_bedrock.sh`. - Dry-run by default (`--execute` flag to run for real) - Step 1 — Request model access: List each model from `aws-design-ai.json` → `bedrock_models[].aws_model_id` and the embedding model -- Step 2 — Create IAM role: Trust policy for the compute platform (Lambda, ECS, or EC2 based on `aws-design.json` if present). Match inference permissions to the selected endpoint. Mantle needs `bedrock-mantle:CreateInference` scoped to `arn:aws:bedrock-mantle:{region}:{account_id}:project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` when using bearer credentials. Runtime needs `bedrock:InvokeModel` / `bedrock:InvokeModelWithResponseStream` against the selected foundation-model ARN and, for CRIS, the exact inference-profile ARN. For EVERY `runtime_openai_cris` row whose `required_client_api` is `responses` or `chat_completions` — classified by id form, not model family: any `us.`/`global.` (and, for GPT-5.6 Terra and Luna, `in.`) `openai.gpt-*` CRIS id, so GPT-5.6 Sol / Terra / Luna, the GPT-6 family, GPT-5.5 and GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default`: a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, "Permissions", read 2026-11-07), and each of those model cards (read 2026-11-07) repeats the requirement for its runtime CRIS ids. `iam_policy.py --target =` (one pair per `bedrock_models[]` row, each row's own API, so a GPT Chat / Responses secondary beside a Claude Messages primary still gets it) emits that third ARN for exactly those rows. A row with `required_client_api: "converse"` (the Converse-only application-inference-profile route) and every Anthropic / Nova / gpt-oss id get no such grant; a bare `openai.gpt-5.5` / `openai.gpt-5.4` / `openai.gpt-5.6-*` / `openai.gpt-6-*` id is a mantle target and gets no such grant. Runtime actions do not authorize Mantle. +- Step 2 — Create IAM role: Trust policy for the compute platform (Lambda, ECS, or EC2 based on `aws-design.json` if present). Match inference permissions to the selected endpoint. Mantle needs `bedrock-mantle:CreateInference` scoped to `arn:aws:bedrock-mantle:{region}:{account_id}:project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` when using bearer credentials. Runtime needs `bedrock:InvokeModel` / `bedrock:InvokeModelWithResponseStream` against the selected foundation-model ARN and, for CRIS, the exact inference-profile ARN. For EVERY `runtime_openai_cris` row whose `required_client_api` is `responses` or `chat_completions` — classified by id form, not model family: any `us.`/`global.` (and, for GPT-5.6 Terra and Luna, `in.`) `openai.gpt-*` CRIS id, so GPT-5.6 Sol / Terra / Luna, the GPT-6 family, GPT-5.5 and GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default`: a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, "Permissions", read 2026-11-07), and each of those model cards (read 2026-11-07) repeats the requirement for its runtime CRIS ids. `iam_policy.py --target =` (one pair per `bedrock_models[]` row, each row's own API, so a GPT Chat / Responses secondary beside a Claude Messages primary still gets it) emits that third ARN for exactly those rows. A row with `required_client_api: "converse"` (the Converse-only application-inference-profile route) and every Anthropic / Nova / gpt-oss id get no such grant; a bare `openai.gpt-5.5` / `openai.gpt-5.4` / `openai.gpt-5.6-*` / `openai.gpt-6-*` id is a mantle target and gets no such grant. Runtime actions do not authorize Mantle. **Retained stored-response operations:** when the source code keeps `client.responses.retrieve(id)` (or `.cancel(id)` / `.delete(id)`; scan for `responses.retrieve(` / `responses.cancel(` / `responses.delete(`), a runtime `responses` row ALSO needs `bedrock:GetInvoke` (and `bedrock:CancelInvoke` / `bedrock:DeleteInvoke` only for the ops actually retained) on the same `arn:aws:bedrock:{region}:{account}:project/default` — creating a response authorizes `bedrock:InvokeModel` on the target and the project, while retrieving, canceling or deleting a stored response authorizes those actions on the project (same page, "Permissions", read 2026-11-08); without them creation and the ping pass and the app's existing retrieve is denied. `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits that `BedrockStoredResponseOps` statement for a runtime `responses` row only and refuses (`reason: stored_response_ops_no_responses_row`) for a `converse`, `messages` or `chat_completions` plan, which has no stored responses. Step 5's verification must then also retrieve the response it just created. - If the runtime client uses `BEDROCK_INFERENCE_PROFILE_ARN` from the generated monitoring output, also authorize that exact `application-inference-profile/…` ARN, retaining the backing model/CRIS resources needed for the selected model and fallback. Resolve it from the same output used by the client; do not grant every application profile. Re-run setup after enabling or changing this override so the effective invocation target and IAM remain aligned. See [inference-profile IAM prerequisites](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-prereq.html). - Compliance carry-through: if `preferences.json` → `design_constraints.compliance` is set and not `none`/`unknown` (full-flow Q2 / AI-only Q1.5), the script and its printed notes MUST reflect the design's Step 0.7 constraints — `AWS_REGION` from the compliance-constrained region (GovCloud for fedramp, EU for gdpr), `eu.` inference-profile model IDs for gdpr, and for hipaa a printed warning that Bedrock invocation logs retain original content (KMS-encrypt + restrict IAM on the log group before enabling) - Step 3 — Print required environment variables: `AWS_REGION`, `AI_PROVIDER=bedrock`, model IDs - Step 4 — Check quota: Query current TPM quota for the primary model via `aws service-quotas get-service-quota`. If `aws-design-ai.json` → `ai_architecture.quota_risk` is `"high"` or `"medium"`, print warning: "⚠️ Your token volume may exceed default Bedrock quotas. Request a quota increase via Service Quotas console (allow 1–5 business days)." Include the `aws service-quotas request-service-quota-increase` command template. -- Step 5 — Verification: Run `test_comparison.py --target-only --quick` using the same selected API, model ID and region as the generated client. A Mantle selection must never run a Converse probe; a runtime selection must preserve its validated profile ID. +- Step 5 — Verification: Run `test_comparison.py --target-only --quick` using the same selected API, model ID and region as the generated client. A Mantle selection must never run a Converse probe; a runtime selection must preserve its validated profile ID. When Step 2 granted stored-response operations, add a create-plus-retrieve check on the runtime Responses API: `r = client.responses.create(model=, input="ping")` then `client.responses.retrieve(r.id)` (and `.cancel` / `.delete` for each other retained op) — a 403 on the second call means the `BedrockStoredResponseOps` grant is missing even though creation passed. - If `$MIGRATION_DIR/terraform/` exists, print coordination note: "Ensure the IAM role is referenced in compute.tf task definitions" - Use region from `preferences.json` → `design_constraints.target_region` diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index 8e874d3eb..11af0d657 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -344,6 +344,16 @@ limited to: `iam_policy.py --target =` (one pair per `bedrock_models[]` row, each row's own API) emits it for `responses` / `chat_completions` rows and omits it for a `converse` or `messages` row and for every non-OpenAI id — a GPT secondary beside a Claude Messages primary still gets it +5. Retained stored-response operations → a source that keeps `client.responses.retrieve(id)` (or `.cancel(id)` / + `.delete(id)`) ALSO needs `bedrock:GetInvoke` (`bedrock:CancelInvoke` / `bedrock:DeleteInvoke`, only for the ops + actually retained) on the same `project/default` ARN: creating a response authorizes `bedrock:InvokeModel` on + the target and the project, while retrieving, canceling or deleting a stored response authorizes those actions + on the project (same page, "Permissions", read 2026-11-08) — without them creation and the preflight ping pass + and the app's existing retrieve is denied. `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits + the `BedrockStoredResponseOps` statement for a runtime `responses` row only and refuses for a `converse` / + `messages` / `chat_completions` plan (`reason: stored_response_ops_no_responses_row`), which has no stored + responses. Verify with create-plus-retrieve: `r = client.responses.create(...)` then + `client.responses.retrieve(r.id)` under the attached role. **Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md b/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md index 87d5bb904..214b5910a 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md @@ -853,10 +853,26 @@ reliably visible, and even that must be materialized to the file for the baselin `required_client_api` (older plans: that row's `runtime_api`, then `ai_architecture.code_migration.runtime_api`) is the API B4, the context block and C0 all use. When the plan records one, `$RESOLVED_RUNTIME_API` is that value. When a `runtime_openai_cris` -plan records none: on resume, read `resolved_runtime_api` from the saved -`$REPO/.saws-migrate/phase-results/run-context.json` (C0's `$PHASE_DIR`) and reuse it without -asking; otherwise AskUserQuestion "Which API does the source app call: -Responses or Chat Completions?" and set `$RESOLVED_RUNTIME_API` to the answer. Other paths that +plan records none, check the saved answer's identity BEFORE reusing it — the saved +`resolved_runtime_api` in `$REPO/.saws-migrate/phase-results/run-context.json` (C0's +`$PHASE_DIR`) belongs to the design it was given for, and C0's `assess_design_sha256` +comparison runs only AFTER these probes: + +```bash +uv run --project $SCRIPTS python $SCRIPTS/preflight_bedrock.py --resume-runtime-api $PHASE_DIR/run-context.json --design $MIGRATION_DIR/aws-design-ai.json +``` + +Exit 0 (`reason: saved_identity_match`) means the saved answer exists AND the saved +`assess_design_sha256` equals the current design's sha256 AND the saved `target_models` +(source model, `aws_model_id`) pairs equal the current `bedrock_models[]` rows: set +`$RESOLVED_RUNTIME_API` to the printed `resolved_runtime_api` and reuse it without asking. +Exit 1 (`no_saved_context`, `no_saved_runtime_api`, or `saved_identity_mismatch` with the +differing fields) means the design, source model or target changed since the answer was +recorded: do NOT read the saved value — AskUserQuestion "Which API does the source app call: +Responses or Chat Completions?" now, before any probe, and set `$RESOLVED_RUNTIME_API` to the +answer (C0 later archives the stale units per its table). A cached `messages` answer must never +send a new GPT row to the Anthropic endpoint, and a stale `responses` answer must never override +a Chat Completions source. Other paths that record no API resolve from the path itself: `runtime_messages` → `messages`; `mantle_openai_chat` → `chat_completions`; `mantle` / `mantle_openai_responses` → `responses` (a bare GPT id is probed on the Mantle Responses API either way); `converse` / `direct` / @@ -948,8 +964,9 @@ stop for an explicit supported API decision rather than treating it as Converse. that explicit API and **fails closed when none is recorded**: `normalize_api_path` raises `ValueError` ("Converse is never the implicit default"). Reaching that error here means B4's resolution step was skipped — do not catch it and guess; go back to B4, resolve -`$RESOLVED_RUNTIME_API` (on resume, read `resolved_runtime_api` from run-context before asking -again), and re-normalize. The same value is written to current-context.json as +`$RESOLVED_RUNTIME_API` (on resume, reuse the saved `resolved_runtime_api` only when B4's +`--resume-runtime-api` gate reports `saved_identity_match`; otherwise ask), and re-normalize. +The same value is written to current-context.json as `resolved_runtime_api` (C0), so an unchanged resume matches without re-asking. An explicit `required_client_api: "converse"` is the ONLY route to a Converse rewrite for a `runtime_openai_cris` plan. `runtime_messages` (Anthropic SDK on the runtime `/anthropic` diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py index 52ded4128..3418a820a 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py @@ -18,6 +18,19 @@ # value preflight accepts for those rows); it never selects the OpenAI-compatible path. TARGET_APIS = RUNTIME_APIS | {"invoke"} +# Stored-response operations a source app may RETAIN after the rewrite, and the action +# each one authorizes on the default project. Creating a response authorizes +# `bedrock:InvokeModel` on the inference target and the project; retrieving, canceling +# and deleting a stored response authorize these three, each on the project +# (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, +# "Permissions", read 2026-11-08). They exist only on the runtime Responses API: +# Converse, Anthropic Messages and Chat Completions rows never get them. +STORED_RESPONSE_OPS = { + "retrieve": "bedrock:GetInvoke", + "cancel": "bedrock:CancelInvoke", + "delete": "bedrock:DeleteInvoke", +} + # Execution lanes this helper refuses to authorize. `mantle_messages` (Anthropic SDK # on the bedrock-mantle `/anthropic` endpoint, selected by the `workspaces` mantle # selector) is deferred: llm-to-bedrock has no preflight, evaluation or rewrite route @@ -200,10 +213,81 @@ def target_apis_by_id(pairs: list[tuple[str, str]]) -> dict[str, set[str]]: return by_id +def parse_stored_response_ops(specs: list[str]) -> list[str]: + """Pure: parse repeatable `--stored-response-ops OP[,OP]` specs into an ordered, + de-duplicated list of `retrieve` / `cancel` / `delete`. Fails CLOSED on any other + value so an action is never granted on a guess.""" + ops: list[str] = [] + bad: list[str] = [] + for spec in specs: + for item in spec.split(","): + item = item.strip() + if not item: + continue + if item not in STORED_RESPONSE_OPS: + bad.append(item) + elif item not in ops: + ops.append(item) + if bad: + raise ValueError( + "Every --stored-response-ops value must be one of " + f"{' | '.join(STORED_RESPONSE_OPS)} (the client.responses.(id) calls the source " + f"app RETAINS). Unknown: {', '.join(bad)}.") + return ops + + +def validated_target_pairs(rows: list[dict], validated_mapping: list[str] | dict[str, str], + primary_model_id: str | None = None, + primary_api: str | None = None) -> list[tuple[str, str]]: + """Pure: build the `--target ID=API` pairs for the report's IAM recipe by JOINING + each `bedrock_models[]` row to the FINAL validated source→target mapping. + + The id comes from `validated_mapping` — the validated analysis `target_models` + entries (`" -> "`, one per row, re-emitted after every + C4 `resolved_model_overrides` change) or an equivalent `{source: target}` dict — + never from the row's own `aws_model_id`, which `aws-design-ai.json` keeps as the + ORIGINAL design id after the user changes the target in C4 (a Sol → Terra change + must emit Terra's profile, not Sol's). The API comes from the row + (`required_client_api`, older plans `runtime_api`); the ONE row whose validated + target is `primary_model_id` may fall back to `primary_api` (B4's resolved surface) + when it records none. Fails CLOSED: a row with no validated mapping entry, or a + row with no API, raises `ValueError` instead of granting on the design's stale id.""" + if isinstance(validated_mapping, dict): + mapping = dict(validated_mapping) + else: + mapping = {} + for entry in validated_mapping: + source, sep, target = entry.partition("->") + if not sep or not source.strip() or not target.strip(): + raise ValueError( + f"validated mapping entry {entry!r} is not ' -> '") + mapping[source.strip()] = target.strip() + pairs: list[tuple[str, str]] = [] + for row in rows: + source = row.get("source_model") or row.get("source_model_id") or "" + if source not in mapping: + raise ValueError( + f"bedrock_models[] row for source {source!r} (design id " + f"{row.get('aws_model_id')!r}) has no entry in the validated target mapping; " + "re-run C1 so analysis.target_models covers every row — never grant the " + "design's original id on a guess.") + target = mapping[source] + api = row.get("required_client_api") or row.get("runtime_api") + if not api and primary_api and target == primary_model_id: + api = primary_api + if not api: + raise ValueError( + f"bedrock_models[] row for source {source!r} → {target!r} records no " + "required_client_api; resolve it in the saved plan instead of guessing it.") + pairs.append((target, api)) + return pairs + + def generate_policy(model_ids: list[str], region: str, account_id: str, runtime_api: str | None = None, migration_path: str | None = None, - target_apis: dict[str, str | set[str]] | None = None) -> dict: + target_apis: dict[str, str | set[str]] | None = None, + stored_response_ops: list[str] | None = None) -> dict: """Build a scoped IAM policy covering exactly the given model IDs. `runtime_api` is the plan's selected API (`responses`, `chat_completions`, @@ -219,8 +303,16 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, UNSUPPORTED lane (`mantle_messages`) raises `ValueError` instead of emitting runtime `bedrock:InvokeModel*` for a bare Claude id that the lane would never send to bedrock-runtime. - - Emits up to four statements, depending on which endpoints the targets use: + `stored_response_ops` lists the stored-response operations the source app + RETAINS (`retrieve` / `cancel` / `delete`, see STORED_RESPONSE_OPS). Each adds + its action — `bedrock:GetInvoke` / `CancelInvoke` / `DeleteInvoke` — on the + default project, which is what retrieving, canceling or deleting a stored + response authorizes (inference-responses-api.html, "Permissions", read + 2026-11-08). Only a runtime GPT CRIS row on `responses` can carry stored + responses, so declaring ops when no such row exists raises `ValueError`: a + Converse, Anthropic Messages or Chat Completions plan never gets these grants. + + Emits up to five statements, depending on which endpoints the targets use: - `bedrock:InvokeModel*` scoped to foundation-model ARNs (plain IDs) and inference-profile ARNs (geo-prefixed IDs), for bedrock-runtime targets - `bedrock:GetInferenceProfile` scoped to the inference-profile ARNs, when any @@ -228,6 +320,8 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, prerequisite for invoking through a profile (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-prereq.html, read 2026-11-07) + - `bedrock:GetInvoke` / `CancelInvoke` / `DeleteInvoke` (only the retained ones) + on the default project ARN, for a runtime Responses row with stored responses - `bedrock-mantle:CreateInference` / `Get*` / `List*` scoped to the account's mantle projects, for mantle-only targets (SigV4 auth) - `bedrock-mantle:CallWithBearerToken` on `*`, for Bedrock API-key auth @@ -252,6 +346,11 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, if api not in TARGET_APIS: raise ValueError(f"Unknown runtime API {api!r} for target {mid!r}; expected one of " f"{sorted(TARGET_APIS)}") + stored_response_ops = list(stored_response_ops or []) + for op in stored_response_ops: + if op not in STORED_RESPONSE_OPS: + raise ValueError(f"Unknown stored-response op {op!r}; expected one of " + f"{sorted(STORED_RESPONSE_OPS)}") runtime_ids = [m for m in model_ids if not is_mantle_model(m, region)] mantle_ids = [m for m in model_ids if is_mantle_model(m, region)] @@ -301,6 +400,29 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, "Resource": sorted(set(profile_arns)), }) + if stored_response_ops: + # Stored responses exist only on the runtime Responses API, so the grant is + # keyed on a row that is BOTH a runtime GPT CRIS target AND on `responses` + # (the same rows that get the default-project InvokeModel grant for + # creation). A Converse / Messages / Chat Completions plan, or a legacy + # id-only call with no API, fails closed instead of receiving GetInvoke. + responses_rows = [ + mid for mid in runtime_ids + if "responses" in (target_apis.get(mid) or {runtime_api}) + and requires_default_project_grant(mid, "responses")] + if not responses_rows: + raise ValueError( + f"stored-response ops {stored_response_ops} declared but no --target row is a " + "runtime GPT CRIS id on the `responses` API; retrieving / canceling / deleting a " + "stored response exists only there, so a converse / messages / chat_completions " + "plan never gets bedrock:GetInvoke / CancelInvoke / DeleteInvoke") + statements.append({ + "Sid": "BedrockStoredResponseOps", + "Effect": "Allow", + "Action": sorted(STORED_RESPONSE_OPS[op] for op in stored_response_ops), + "Resource": [default_project_arn(region, account_id)], + }) + if mantle_ids: statements.append({ "Sid": "BedrockMantleInference", @@ -349,6 +471,12 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, parser.add_argument("--migration-path", help="Plan's code_migration.migration_path; an unsupported lane " "(mantle_messages) exits 1 with a JSON reason instead of a policy.") + parser.add_argument("--stored-response-ops", action="append", default=[], + help="retrieve | cancel | delete (comma-joined or repeated): the " + "client.responses.(id) calls the source app RETAINS. Adds " + "bedrock:GetInvoke / CancelInvoke / DeleteInvoke on project/default " + "for a runtime GPT CRIS row on `responses`; exits 1 with reason " + "stored_response_ops_no_responses_row when no such row exists.") args = parser.parse_args() def _fail(reason: str, detail: str) -> None: @@ -372,12 +500,18 @@ def _fail(reason: str, detail: str) -> None: target_apis = target_apis_by_id(targets) else: model_ids = [m.strip() for m in args.models.split(",") if m.strip()] + try: + stored_response_ops = parse_stored_response_ops(args.stored_response_ops) + except ValueError as e: + _fail("unknown_stored_response_op", str(e)) try: policy = generate_policy(model_ids, args.region, args.account_id, runtime_api=args.runtime_api if not args.target else None, - migration_path=args.migration_path, target_apis=target_apis) + migration_path=args.migration_path, target_apis=target_apis, + stored_response_ops=stored_response_ops) except ValueError as e: - _fail("unknown_runtime_api", str(e)) + _fail("stored_response_ops_no_responses_row" if "stored-response ops" in str(e) + else "unknown_runtime_api", str(e)) output = json.dumps(policy, indent=2) + "\n" if args.output: diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py index c6effec95..13b40338b 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/preflight_bedrock.py @@ -5,6 +5,7 @@ python preflight_bedrock.py --region --target = [--target ...] [--dataset-size N] python preflight_bedrock.py --region --models [--runtime-api ] [--dataset-size N] python preflight_bedrock.py --check-lanes + python preflight_bedrock.py --resume-runtime-api --design Prints a JSON verdict to stdout; exit 0 if all models pass, 1 otherwise. Always prints JSON — including for missing credentials / bad region — so the caller can parse the verdict instead of a traceback. On failure the top level @@ -22,6 +23,12 @@ `reason: unsupported_lane` when any row's `migration_path` is a deferred lane (`mantle_messages`, the Workspaces selector's Anthropic-SDK-on-Mantle route); run it before preflight or any Phase C dispatch. +`--resume-runtime-api` is B4's resume gate for a `runtime_openai_cris` plan that records +no API: the saved `resolved_runtime_api` is reused (exit 0) only when the saved +`assess_design_sha256` and `target_models` identity still match the current design; +otherwise (exit 1) the current row's API is resolved BEFORE any probe, so a cached +`messages` answer never sends a new GPT row to the Anthropic endpoint and a stale +`responses` answer never overrides a Chat Completions source. Embedding models (which don't speak Converse) go via InvokeModel with their family's request body. Bare OpenAI proprietary GPT ids are the bedrock-mantle form and are probed via the OpenAI Responses API on that endpoint; GPT-6 Astra @@ -409,6 +416,79 @@ def check_lanes(design_path: str) -> int: return 0 +def design_row_identity(rows: list) -> list[tuple[str, str]]: + """Pure: the (source model, aws_model_id) identity of every design row, in row order. + Accepts the design's `source_model_id` spelling and run-context's `source_model`.""" + return [(str(r.get("source_model") or r.get("source_model_id") or ""), + str(r.get("aws_model_id") or "")) for r in rows] + + +def resume_runtime_api(saved_context: dict | None, design_sha256: str, rows: list) -> dict: + """Pure: decide whether B4 may REUSE the saved `resolved_runtime_api` on a resume + without asking again, or must resolve the current API before any probe. + + The saved answer belongs to the design it was given for. It is reused only when it + exists AND the saved identity still matches the current inputs: `assess_design_sha256` + equals the sha256 of the current `aws-design-ai.json`, and the saved `target_models` + (source_model, aws_model_id) pairs equal the current `bedrock_models[]` rows. A changed + design, a different source model, a different target id, or a saved context without + the field all return `ok: false` — a cached `messages` answer must never send a new + GPT row to the Anthropic endpoint, and a stale `responses` answer must never override + a Chat Completions source. C0's own identity check runs later; this gate is what + keeps B4's probes from dispatching on the previous design's answer first.""" + if not saved_context: + return {"ok": False, "reason": "no_saved_context", + "detail": "No saved run-context.json: resolve the current row's API before B4 probes."} + saved_api = saved_context.get("resolved_runtime_api") + if not saved_api: + return {"ok": False, "reason": "no_saved_runtime_api", + "detail": "Saved run-context records no resolved_runtime_api; resolve it now."} + mismatches = [] + saved_sha = saved_context.get("assess_design_sha256") + if saved_sha != design_sha256: + mismatches.append(f"assess_design_sha256 saved={saved_sha!r} current={design_sha256!r}") + saved_rows = design_row_identity(saved_context.get("target_models") or []) + current_rows = design_row_identity(rows) + if saved_rows != current_rows: + mismatches.append(f"target_models saved={saved_rows!r} current={current_rows!r}") + if mismatches: + return {"ok": False, "reason": "saved_identity_mismatch", "mismatches": mismatches, + "detail": ("Saved resolved_runtime_api belongs to a different design/source/target; " + "resolve the current row's API before B4 probes (C0 will archive per its table).")} + return {"ok": True, "reason": "saved_identity_match", "resolved_runtime_api": saved_api, + "detail": "Saved design, source and target identity match; reuse without asking."} + + +def check_resume_runtime_api(run_context_path: str, design_path: str) -> int: + """CLI entry for the B4 resume gate: print JSON, exit 0 when the saved API may be + reused, 1 when the current API must be resolved first.""" + import hashlib + try: + with open(design_path, "rb") as fh: + design_bytes = fh.read() + design = json.loads(design_bytes.decode("utf-8")) + except (OSError, ValueError) as e: + print(json.dumps({"ok": False, "reason": "design_unreadable", + "detail": f"{type(e).__name__}: {e}"}, indent=2)) + return 1 + saved = None + try: + with open(run_context_path, encoding="utf-8") as fh: + saved = json.load(fh) + except FileNotFoundError: + saved = None + except (OSError, ValueError) as e: + print(json.dumps({"ok": False, "reason": "run_context_unreadable", + "detail": f"{type(e).__name__}: {e}"}, indent=2)) + return 1 + arch = design.get("ai_architecture", {}) if isinstance(design, dict) else {} + rows = arch.get("bedrock_models") or [] + verdict = resume_runtime_api(saved if isinstance(saved, dict) else None, + hashlib.sha256(design_bytes).hexdigest(), rows) + print(json.dumps(verdict, indent=2)) + return 0 if verdict["ok"] else 1 + + def parse_targets(specs: list[str]) -> tuple[list[tuple[str, str]], dict | None]: """Pure: parse `--target ID=API` specs into (model_id, api) pairs. @@ -549,6 +629,13 @@ def main(argv=None) -> int: group.add_argument("--check-lanes", metavar="AWS_DESIGN_AI_JSON", help="stop with reason unsupported_lane when any bedrock_models[] row " "selects a deferred lane (mantle_messages); probes nothing") + group.add_argument("--resume-runtime-api", metavar="RUN_CONTEXT_JSON", + help="B4 resume gate: exit 0 and print the saved resolved_runtime_api " + "only when the saved assess_design_sha256 and target_models still " + "match --design; exit 1 (reason saved_identity_mismatch / " + "no_saved_context) when the current API must be resolved first") + ap.add_argument("--design", metavar="AWS_DESIGN_AI_JSON", + help="current aws-design-ai.json for --resume-runtime-api") ap.add_argument("--dataset-size", type=int, default=0) ap.add_argument("--runtime-api", choices=["converse", "responses", "chat_completions", "messages"], @@ -558,6 +645,10 @@ def main(argv=None) -> int: if args.check_lanes: return check_lanes(args.check_lanes) + if args.resume_runtime_api: + if not args.design: + ap.error("--resume-runtime-api requires --design ") + return check_resume_runtime_api(args.resume_runtime_api, args.design) if not args.region or not (args.models or args.target): ap.error("--region and one of --models / --target are required") diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/schemas/analysis.json b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/schemas/analysis.json index b155be841..914b176c7 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/schemas/analysis.json +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/schemas/analysis.json @@ -73,7 +73,13 @@ } } }, - "source_baseline_available": { "type": "boolean" } + "source_baseline_available": { "type": "boolean" }, + "stored_response_ops": { + "$comment": "Optional (analyzer §14): the client.responses.(id) calls the source app makes and the rewrite RETAINS — retrieve / cancel / delete. The report's §6.4 IAM recipe passes them to iam_policy.py --stored-response-ops so a runtime Responses row gets bedrock:GetInvoke / CancelInvoke / DeleteInvoke on project/default. Absent or [] = none retained.", + "type": "array", + "uniqueItems": true, + "items": { "type": "string", "enum": ["retrieve", "cancel", "delete"] } + } } }, { diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy_round2.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy_round2.py new file mode 100644 index 000000000..af7a203c0 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy_round2.py @@ -0,0 +1,190 @@ +"""PR 585 review round 2 — iam_policy.py helper behaviour. + +Finding 1: the report's §6.4 recipe re-read `aws-design-ai.json`'s ORIGINAL ids, so a C4 +target change (Sol → Terra, recorded in `resolved_model_overrides` and carried by the +validated analysis `target_models`) emitted Sol's permissions and omitted Terra's profile. +`validated_target_pairs` joins each row's API to the final validated mapping. + +Finding 2: a source app that keeps `client.responses.retrieve(id)` needs `bedrock:GetInvoke` +on `project/default` — retrieving / canceling / deleting a stored response authorizes +`bedrock:GetInvoke` / `CancelInvoke` / `DeleteInvoke` on the project, separately from the +`bedrock:InvokeModel` creation needs (public basis, read 2026-11-08: +https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, +"Permissions"). Converse, Anthropic Messages and Chat Completions rows never get them. +""" +import json +import subprocess # nosec B404 — test-only, inputs are hardcoded literals +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent)) + +from iam_policy import ( # noqa: E402 + STORED_RESPONSE_OPS, + default_project_arn, + generate_policy, + inference_profile_arn, + parse_stored_response_ops, + parse_targets, + target_apis_by_id, + validated_target_pairs, +) + +REGION = "us-east-1" +ACCOUNT = "123456789012" +PROJECT = default_project_arn(REGION, ACCOUNT) +SOL, TERRA = "us.openai.gpt-5.6-sol", "us.openai.gpt-5.6-terra" +CLAUDE = "global.anthropic.claude-sonnet-5" +EMBED = "amazon.titan-embed-text-v2:0" + + +def _sids(pol): + return [s["Sid"] for s in pol["Statement"]] + + +def _stmt(pol, sid): + return {s["Sid"]: s for s in pol["Statement"]}[sid] + + +# --- Finding 1: validated mapping wins over the design row's original id --- + +class TestValidatedTargetPairs: + DESIGN_ROWS = [ + {"source_model_id": "gpt-5.6-sol", "aws_model_id": SOL, "required_client_api": "responses"}, + {"source_model_id": "text-embedding-3-small", "aws_model_id": EMBED, "required_client_api": "invoke"}, + ] + + def test_c4_target_change_emits_the_new_profile_and_not_the_old(self): + # The reviewer's case: C4 changed Sol → Terra; aws-design-ai.json still says Sol. + validated = [f"gpt-5.6-sol -> {TERRA}", f"text-embedding-3-small -> {EMBED}"] + pairs = validated_target_pairs(self.DESIGN_ROWS, validated) + assert pairs == [(TERRA, "responses"), (EMBED, "invoke")] + pol = generate_policy([m for m, _ in pairs], REGION, ACCOUNT, target_apis=target_apis_by_id(pairs)) + resources = _stmt(pol, "BedrockInvokeModelScoped")["Resource"] + assert inference_profile_arn(TERRA, REGION, ACCOUNT) in resources + assert "arn:aws:bedrock:*::foundation-model/openai.gpt-5.6-terra" in resources + assert PROJECT in resources, "the Terra Responses row keeps the default-project grant" + assert not any("gpt-5.6-sol" in r for r in resources), "the original Sol id must not be granted" + assert _stmt(pol, "BedrockGetInferenceProfile")["Resource"] == [inference_profile_arn(TERRA, REGION, ACCOUNT)] + + def test_unchanged_mapping_reproduces_the_design_rows(self): + validated = {"gpt-5.6-sol": SOL, "text-embedding-3-small": EMBED} + assert validated_target_pairs(self.DESIGN_ROWS, validated) == [(SOL, "responses"), (EMBED, "invoke")] + + def test_primary_api_fills_only_the_primary_row_without_an_api(self): + rows = [{"source_model": "gpt-5.6-sol", "aws_model_id": SOL}, + {"source_model": "claude-sonnet-4-5", "aws_model_id": CLAUDE, "required_client_api": "messages"}] + pairs = validated_target_pairs(rows, [f"gpt-5.6-sol -> {TERRA}", f"claude-sonnet-4-5 -> {CLAUDE}"], + primary_model_id=TERRA, primary_api="chat_completions") + assert pairs == [(TERRA, "chat_completions"), (CLAUDE, "messages")] + + def test_non_primary_row_without_an_api_fails_closed(self): + rows = [{"source_model": "gpt-5.6-sol", "aws_model_id": SOL, "required_client_api": "responses"}, + {"source_model": "claude-sonnet-4-5", "aws_model_id": CLAUDE}] + with pytest.raises(ValueError, match="records no required_client_api"): + validated_target_pairs(rows, [f"gpt-5.6-sol -> {SOL}", f"claude-sonnet-4-5 -> {CLAUDE}"], + primary_model_id=SOL, primary_api="responses") + + def test_row_without_a_validated_entry_fails_closed_instead_of_using_the_design_id(self): + with pytest.raises(ValueError, match="no entry in the validated target mapping"): + validated_target_pairs(self.DESIGN_ROWS, [f"gpt-5.6-sol -> {TERRA}"]) + + def test_malformed_mapping_entry_is_rejected(self): + with pytest.raises(ValueError, match="is not ' -> '"): + validated_target_pairs(self.DESIGN_ROWS[:1], ["gpt-5.6-sol"]) + + def test_pairs_round_trip_through_the_cli_target_form(self): + pairs = validated_target_pairs(self.DESIGN_ROWS, [f"gpt-5.6-sol -> {TERRA}", f"text-embedding-3-small -> {EMBED}"]) + assert parse_targets([f"{m}={a}" for m, a in pairs]) == pairs + + +# --- Finding 2: retained stored-response ops get project-scoped GetInvoke / CancelInvoke / DeleteInvoke --- + +class TestStoredResponseOps: + def test_retained_retrieve_adds_get_invoke_on_the_default_project_only(self): + pol = generate_policy([SOL], REGION, ACCOUNT, target_apis={SOL: "responses"}, + stored_response_ops=["retrieve"]) + assert _sids(pol) == ["BedrockInvokeModelScoped", "BedrockGetInferenceProfile", "BedrockStoredResponseOps"] + stmt = _stmt(pol, "BedrockStoredResponseOps") + assert stmt["Action"] == ["bedrock:GetInvoke"] + assert stmt["Resource"] == [PROJECT] + # Creation still authorizes InvokeModel on the target AND the project. + assert PROJECT in _stmt(pol, "BedrockInvokeModelScoped")["Resource"] + + def test_only_the_retained_ops_are_granted(self): + pol = generate_policy([SOL], REGION, ACCOUNT, target_apis={SOL: "responses"}, + stored_response_ops=["retrieve", "delete"]) + assert _stmt(pol, "BedrockStoredResponseOps")["Action"] == ["bedrock:DeleteInvoke", "bedrock:GetInvoke"] + full = generate_policy([SOL], REGION, ACCOUNT, target_apis={SOL: "responses"}, + stored_response_ops=["retrieve", "cancel", "delete"]) + assert _stmt(full, "BedrockStoredResponseOps")["Action"] == sorted(STORED_RESPONSE_OPS.values()) + + def test_no_ops_means_no_stored_response_statement(self): + pol = generate_policy([SOL], REGION, ACCOUNT, target_apis={SOL: "responses"}) + assert "BedrockStoredResponseOps" not in _sids(pol) + assert not any("GetInvoke" in a for s in pol["Statement"] for a in s["Action"]) + + @pytest.mark.parametrize("target_apis,ids", [ + ({CLAUDE: "messages"}, [CLAUDE]), + ({"us.amazon.nova-lite-v1:0": "converse"}, ["us.amazon.nova-lite-v1:0"]), + ({SOL: "converse"}, [SOL]), + ({SOL: "chat_completions"}, [SOL]), + ({CLAUDE: "messages", EMBED: "invoke"}, [CLAUDE, EMBED]), + ]) + def test_converse_messages_chat_and_embedding_rows_never_get_the_grant(self, target_apis, ids): + with pytest.raises(ValueError, match="stored-response ops .* no --target row is a runtime GPT CRIS id on the `responses` API"): + generate_policy(ids, REGION, ACCOUNT, target_apis=target_apis, stored_response_ops=["retrieve"]) + + def test_mixed_plan_grants_once_for_the_responses_row_beside_a_messages_primary(self): + pol = generate_policy([CLAUDE, SOL], REGION, ACCOUNT, migration_path="runtime_messages", + target_apis={CLAUDE: "messages", SOL: "responses"}, stored_response_ops=["retrieve"]) + assert _stmt(pol, "BedrockStoredResponseOps") == { + "Sid": "BedrockStoredResponseOps", "Effect": "Allow", + "Action": ["bedrock:GetInvoke"], "Resource": [PROJECT]} + + def test_bare_mantle_id_on_responses_never_gets_runtime_get_invoke(self): + with pytest.raises(ValueError, match="stored-response ops"): + generate_policy(["openai.gpt-5.6-sol"], REGION, ACCOUNT, target_apis={"openai.gpt-5.6-sol": "responses"}, + stored_response_ops=["retrieve"]) + + def test_legacy_models_form_needs_an_explicit_responses_api(self): + pol = generate_policy([SOL], REGION, ACCOUNT, runtime_api="responses", stored_response_ops=["retrieve"]) + assert _stmt(pol, "BedrockStoredResponseOps")["Action"] == ["bedrock:GetInvoke"] + with pytest.raises(ValueError, match="stored-response ops"): + generate_policy([SOL], REGION, ACCOUNT, stored_response_ops=["retrieve"]) # id-only: API unknown + + def test_unknown_op_fails_closed(self): + with pytest.raises(ValueError, match="Unknown stored-response op 'list'"): + generate_policy([SOL], REGION, ACCOUNT, target_apis={SOL: "responses"}, stored_response_ops=["list"]) + with pytest.raises(ValueError, match="Every --stored-response-ops value must be"): + parse_stored_response_ops(["retrieve,list"]) + + def test_parse_stored_response_ops_dedupes_and_keeps_order(self): + assert parse_stored_response_ops(["retrieve,cancel", "retrieve", " delete "]) == ["retrieve", "cancel", "delete"] + assert parse_stored_response_ops([]) == [] + + def _cli(self, *extra): + return subprocess.run( # nosec B603 + [sys.executable, str(Path(__file__).parent / "iam_policy.py"), + "--region", REGION, "--account-id", ACCOUNT, *extra], + capture_output=True, text=True) + + def test_cli_stored_response_ops_emit_the_statement_for_a_responses_row(self): + result = self._cli("--target", f"{SOL}=responses", "--stored-response-ops", "retrieve,cancel") + assert result.returncode == 0, result.stderr + stmt = _stmt(json.loads(result.stdout), "BedrockStoredResponseOps") + assert stmt["Action"] == ["bedrock:CancelInvoke", "bedrock:GetInvoke"] and stmt["Resource"] == [PROJECT] + + def test_cli_refuses_stored_response_ops_for_a_converse_plan(self): + result = self._cli("--target", f"{SOL}=converse", "--stored-response-ops", "retrieve") + assert result.returncode == 1 + out = json.loads(result.stdout) + assert out["ok"] is False and out["reason"] == "stored_response_ops_no_responses_row" + assert "Statement" not in result.stdout + + def test_cli_refuses_an_unknown_op(self): + result = self._cli("--target", f"{SOL}=responses", "--stored-response-ops", "list") + assert result.returncode == 1 + assert json.loads(result.stdout)["reason"] == "unknown_stored_response_op" diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_resume_runtime_api.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_resume_runtime_api.py new file mode 100644 index 000000000..b8b95c61f --- /dev/null +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_preflight_resume_runtime_api.py @@ -0,0 +1,108 @@ +"""PR 585 review round 2, finding 3 — B4's saved-API reuse must check identity first. + +The old recovery guidance read the previous design's saved `resolved_runtime_api` before C0 +compared `assess_design_sha256`, so a cached `messages` answer could send a new GPT row to the +Anthropic endpoint and a stale `responses` answer could override a Chat Completions source. +`resume_runtime_api` reuses the saved answer only when it exists AND the saved design sha256 +and (source, target) row identity still match; anything else forces re-resolution BEFORE the +probes. +""" +import hashlib +import json + +import preflight_bedrock as p + +DESIGN_SHA = "a" * 64 +GPT_ROWS = [{"source_model_id": "gpt-5.6-sol", "aws_model_id": "us.openai.gpt-5.6-sol", + "migration_path": "runtime_openai_cris"}] +CLAUDE_ROWS = [{"source_model_id": "claude-sonnet-4-5", "aws_model_id": "global.anthropic.claude-sonnet-5", + "migration_path": "runtime_messages"}] + + +def _saved(api, sha=DESIGN_SHA, rows=GPT_ROWS): + return {"resolved_runtime_api": api, "assess_design_sha256": sha, + "target_models": [{"source_model": r["source_model_id"], "aws_model_id": r["aws_model_id"], + "use_case": "chat"} for r in rows]} + + +def test_unchanged_identity_reuses_the_saved_answer_without_asking(): + verdict = p.resume_runtime_api(_saved("chat_completions"), DESIGN_SHA, GPT_ROWS) + assert verdict == {"ok": True, "reason": "saved_identity_match", "resolved_runtime_api": "chat_completions", + "detail": "Saved design, source and target identity match; reuse without asking."} + + +def test_cached_messages_answer_is_not_reused_for_a_new_gpt_row(): + # The reviewer's first case: the previous design was a Claude Messages plan; the design + # now holds a GPT row with no API. The saved `messages` must not reach the GPT probe. + saved = _saved("messages", rows=CLAUDE_ROWS) + verdict = p.resume_runtime_api(saved, "b" * 64, GPT_ROWS) + assert verdict["ok"] is False and verdict["reason"] == "saved_identity_mismatch" + assert "resolved_runtime_api" not in verdict + assert any(m.startswith("assess_design_sha256") for m in verdict["mismatches"]) + assert any(m.startswith("target_models") for m in verdict["mismatches"]) + + +def test_stale_responses_answer_is_not_reused_when_only_the_design_changed(): + # The reviewer's second case: same target id, but the design (and so the source's API) + # changed — a stale `responses` must not override a Chat Completions source. + verdict = p.resume_runtime_api(_saved("responses"), "c" * 64, GPT_ROWS) + assert verdict["ok"] is False and verdict["reason"] == "saved_identity_mismatch" + assert verdict["mismatches"] == [f"assess_design_sha256 saved={DESIGN_SHA!r} current={'c' * 64!r}"] + + +def test_changed_target_id_alone_forces_re_resolution(): + terra_rows = [{**GPT_ROWS[0], "aws_model_id": "us.openai.gpt-5.6-terra"}] + verdict = p.resume_runtime_api(_saved("responses"), DESIGN_SHA, terra_rows) + assert verdict["ok"] is False and len(verdict["mismatches"]) == 1 + assert verdict["mismatches"][0].startswith("target_models ") + + +def test_changed_source_model_alone_forces_re_resolution(): + rows = [{**GPT_ROWS[0], "source_model_id": "gpt-5.5"}] + assert p.resume_runtime_api(_saved("responses"), DESIGN_SHA, rows)["reason"] == "saved_identity_mismatch" + + +def test_missing_context_or_missing_field_never_reuses(): + assert p.resume_runtime_api(None, DESIGN_SHA, GPT_ROWS)["reason"] == "no_saved_context" + assert p.resume_runtime_api({}, DESIGN_SHA, GPT_ROWS)["reason"] == "no_saved_context" + saved = _saved("responses") + del saved["resolved_runtime_api"] + assert p.resume_runtime_api(saved, DESIGN_SHA, GPT_ROWS)["reason"] == "no_saved_runtime_api" + + +def test_design_row_identity_accepts_both_source_key_spellings(): + assert p.design_row_identity(GPT_ROWS) == p.design_row_identity( + [{"source_model": "gpt-5.6-sol", "aws_model_id": "us.openai.gpt-5.6-sol"}]) + + +def _write_design(tmp_path, rows): + design = tmp_path / "aws-design-ai.json" + design.write_text(json.dumps({"ai_architecture": {"bedrock_models": rows}})) + return design, hashlib.sha256(design.read_bytes()).hexdigest() + + +def test_cli_match_exits_zero_and_prints_the_saved_api(tmp_path, capsys): + design, sha = _write_design(tmp_path, GPT_ROWS) + ctx = tmp_path / "run-context.json" + ctx.write_text(json.dumps(_saved("chat_completions", sha=sha))) + rc = p.main(["--resume-runtime-api", str(ctx), "--design", str(design)]) + out = json.loads(capsys.readouterr().out) + assert rc == 0 and out["ok"] is True and out["resolved_runtime_api"] == "chat_completions" + + +def test_cli_identity_mismatch_exits_one_before_any_probe(tmp_path, capsys, monkeypatch): + import boto3 + monkeypatch.setattr(boto3, "client", lambda *a, **k: (_ for _ in ()).throw(AssertionError("probed"))) + design, _ = _write_design(tmp_path, GPT_ROWS) + ctx = tmp_path / "run-context.json" + ctx.write_text(json.dumps(_saved("messages", sha="0" * 64, rows=CLAUDE_ROWS))) + rc = p.main(["--resume-runtime-api", str(ctx), "--design", str(design)]) + out = json.loads(capsys.readouterr().out) + assert rc == 1 and out["reason"] == "saved_identity_mismatch" + assert "resolved_runtime_api" not in out + + +def test_cli_without_a_saved_context_exits_one(tmp_path, capsys): + design, _ = _write_design(tmp_path, GPT_ROWS) + rc = p.main(["--resume-runtime-api", str(tmp_path / "missing.json"), "--design", str(design)]) + assert rc == 1 and json.loads(capsys.readouterr().out)["reason"] == "no_saved_context" diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md index 8e874d3eb..11af0d657 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md @@ -344,6 +344,16 @@ limited to: `iam_policy.py --target =` (one pair per `bedrock_models[]` row, each row's own API) emits it for `responses` / `chat_completions` rows and omits it for a `converse` or `messages` row and for every non-OpenAI id — a GPT secondary beside a Claude Messages primary still gets it +5. Retained stored-response operations → a source that keeps `client.responses.retrieve(id)` (or `.cancel(id)` / + `.delete(id)`) ALSO needs `bedrock:GetInvoke` (`bedrock:CancelInvoke` / `bedrock:DeleteInvoke`, only for the ops + actually retained) on the same `project/default` ARN: creating a response authorizes `bedrock:InvokeModel` on + the target and the project, while retrieving, canceling or deleting a stored response authorizes those actions + on the project (same page, "Permissions", read 2026-11-08) — without them creation and the preflight ping pass + and the app's existing retrieve is denied. `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits + the `BedrockStoredResponseOps` statement for a runtime `responses` row only and refuses for a `converse` / + `messages` / `chat_completions` plan (`reason: stored_response_ops_no_responses_row`), which has no stored + responses. Verify with create-plus-retrieve: `r = client.responses.create(...)` then + `client.responses.retrieve(r.id)` under the attached role. **Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, diff --git a/plugins/aws-startup-advisor/tests/test_runtime_default_review_round2.py b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round2.py new file mode 100644 index 000000000..2dc4c86f1 --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round2.py @@ -0,0 +1,241 @@ +"""Guards for the PR 585 review-round-2 findings (head f3c01556). + +Public basis (read 2026-11-08): + https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html + ("Permissions": creating a response authorizes bedrock:InvokeModel on the inference + target AND the default project; retrieving, canceling and deleting a stored response + authorize bedrock:GetInvoke / CancelInvoke / DeleteInvoke on the project) + +1. Report §6.4 IAM recipe: the `--target` ids come from the VALIDATED analysis mapping + (C4 `resolved_model_overrides` applied), not from aws-design-ai.json's original rows — + a Sol → Terra change in C4 must yield a C6 policy naming Terra and not Sol. +2. Retained stored-response reads: analyzer records `stored_response_ops`, the recipe + carries them to `iam_policy.py --stored-response-ops`, the GCP / Azure setup and the + shared OpenAI guide name the project-scoped GetInvoke grant, and the report checklist + has a create-plus-retrieve check. Converse / Messages never get the grant. +3. B4 resume: the saved `resolved_runtime_api` is reused only after the saved design / + source / target identity is checked (`preflight_bedrock.py --resume-runtime-api`). + +Helper behaviour is tested next to the helpers in +`skills/llm-to-bedrock/scripts/test_iam_policy_round2.py` and +`test_preflight_resume_runtime_api.py`; this file covers the prose contracts and the +end-to-end C4-change → C6-policy regression through the documented recipe. +""" +from __future__ import annotations + +import importlib.util +import json +import re +import sys +from pathlib import Path + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +SKILLS = PLUGIN_ROOT / "skills" +SKILL_MD = SKILLS / "llm-to-bedrock" / "SKILL.md" +SCRIPTS = SKILLS / "llm-to-bedrock" / "scripts" +ANALYZER = PLUGIN_ROOT / "agents" / "llm2bedrock-code-analyzer.md" +REPORT_GENERATOR = PLUGIN_ROOT / "agents" / "llm2bedrock-report-generator.md" +ANALYSIS_SCHEMA = SCRIPTS / "schemas" / "analysis.json" +GCP_GENERATE_ARTIFACTS = SKILLS / "gcp-to-aws" / "references" / "phases" / "generate" / "generate-artifacts-ai.md" +GCP_GENERATE_AI = SKILLS / "gcp-to-aws" / "references" / "phases" / "generate" / "generate-ai.md" +AZURE_GENERATE_ARTIFACTS = SKILLS / "azure-to-aws" / "references" / "phases" / "generate" / "generate-artifacts-ai.md" +CANONICAL_GUIDE = SKILLS / "shared" / "ai" / "ai-openai-to-bedrock.md" +VENDORED_GUIDES = [ + SKILLS / "gcp-to-aws" / "references" / "vendored" / "ai" / "ai-openai-to-bedrock.md", + SKILLS / "azure-to-aws" / "references" / "vendored" / "ai" / "ai-openai-to-bedrock.md", +] +RESPONSES_PAGE = "https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html" + + +def _load(name: str): + if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + spec = importlib.util.spec_from_file_location(name, SCRIPTS / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore[union-attr] + return module + + +def _section(text: str, start: str, end: str) -> str: + i = text.index(start) + return text[i: text.index(end, i)] + + +def _recipe() -> tuple[str, str]: + raw = _section(REPORT_GENERATOR.read_text(), "# 6.4 Generate scoped IAM policy artifact", "# 7. Generate report") + return " ".join(raw.split()), _section(raw, "```bash", "```\n") + + +# --- Finding 1: the recipe joins plan rows to the validated mapping, overrides applied --- + +def test_report_recipe_sources_ids_from_the_validated_mapping_not_the_design_rows() -> None: + recipe, code = _recipe() + assert "--target =" in code + assert "is the VALIDATED target id for that row, not the design row's own `aws_model_id`" in recipe + assert "resolved_model_overrides" in recipe + assert "keeps the ORIGINAL design id after the user changes a target in C4" in recipe + assert "Sol → Terra change would grant `us.openai.gpt-5.6-sol` and omit the Terra profile" in recipe + assert "validated `target_models`" in recipe and "FINAL validated id with every override applied" in recipe + assert "iam_policy.validated_target_pairs(" in recipe + assert "MUST equal the set on your context's `Target Bedrock model(s)` line" in recipe + assert "never fall back to the row's original `aws_model_id`" in recipe + # The inputs section names target_models as the join source. + inputs = _section(REPORT_GENERATOR.read_text(), "# 3. Inputs from orchestrator", "# 4. Skills to load") + assert "FINAL validated ids §6.4's IAM recipe joins the plan rows to" in inputs + + +def test_c4_target_change_flows_into_the_c6_policy_through_the_documented_recipe() -> None: + """The reviewer's regression: C4 changes Sol → Terra. aws-design-ai.json still holds + Sol; the validated analysis holds Terra. The recipe's helper must emit Terra's + profile and the default-project grant, and no Sol permission at all.""" + iam = _load("iam_policy") + design_rows = [{"source_model_id": "gpt-5.6-sol", "aws_model_id": "us.openai.gpt-5.6-sol", + "required_client_api": "responses", "migration_path": "runtime_openai_cris"}] + # C4 recorded {"us.openai.gpt-5.6-sol": "us.openai.gpt-5.6-terra"}; C1 re-ran and validated: + analysis_target_models = ["gpt-5.6-sol -> us.openai.gpt-5.6-terra"] + pairs = iam.validated_target_pairs(design_rows, analysis_target_models, + primary_model_id="us.openai.gpt-5.6-terra", primary_api="responses") + assert pairs == [("us.openai.gpt-5.6-terra", "responses")] + policy = iam.generate_policy([m for m, _ in pairs], "us-east-1", "123456789012", + migration_path="runtime_openai_responses", + target_apis=iam.target_apis_by_id(pairs)) + dumped = json.dumps(policy) + assert "gpt-5.6-terra" in dumped and "gpt-5.6-sol" not in dumped + invoke = {s["Sid"]: s for s in policy["Statement"]}["BedrockInvokeModelScoped"]["Resource"] + assert iam.inference_profile_arn("us.openai.gpt-5.6-terra", "us-east-1", "123456789012") in invoke + assert iam.default_project_arn("us-east-1", "123456789012") in invoke + # Control: the pre-fix behaviour (re-reading the design row) is what emitted Sol. + stale = iam.generate_policy([r["aws_model_id"] for r in design_rows], "us-east-1", "123456789012", + target_apis={"us.openai.gpt-5.6-sol": "responses"}) + assert "gpt-5.6-sol" in json.dumps(stale) and "gpt-5.6-terra" not in json.dumps(stale) + + +# --- Finding 2: retained stored-response ops → project-scoped GetInvoke, propagated everywhere --- + +def test_analysis_schema_and_analyzer_carry_stored_response_ops() -> None: + schema = json.loads(ANALYSIS_SCHEMA.read_text()) + payload = schema["oneOf"][0] + prop = payload["properties"]["stored_response_ops"] + assert "stored_response_ops" not in payload["required"], "optional: older analyses stay valid" + assert prop["items"]["enum"] == ["retrieve", "cancel", "delete"] + text = ANALYZER.read_text() + scan = _section(text, "## 8.3 Detect special patterns", "# 9. Detect behavior deltas") + assert "responses\\.retrieve(" in scan and "responses\\.cancel(" in scan and "responses\\.delete(" in scan + joined = " ".join(scan.split()) + assert "`stored_response_ops`" in joined + assert "bedrock:GetInvoke` / `bedrock:CancelInvoke` / `bedrock:DeleteInvoke` on the account's default project" in joined + assert RESPONSES_PAGE in joined and "read 2026-11-08" in joined + assert "Creation (`responses.create`) is NOT one of them" in joined + assert "never emit it for a Chat Completions, Converse or Anthropic Messages source" in joined + completion = _section(text, "# 14. Completion", "## Example result") + assert "`stored_response_ops` (§8.3) is the one optional typed field" in completion + + +def test_report_recipe_carries_stored_response_ops_to_the_helper_with_controls() -> None: + recipe, code = _recipe() + assert "[--stored-response-ops ]" in code + assert "Retained stored-response operations need their own grant" in recipe + assert "`BedrockStoredResponseOps` statement" in recipe + assert "`bedrock:GetInvoke` for `retrieve`, `bedrock:CancelInvoke` for `cancel`, `bedrock:DeleteInvoke` for `delete`" in recipe + assert RESPONSES_PAGE in recipe and "read 2026-11-08" in recipe + assert "creation and the preflight ping pass while the app's existing `responses.retrieve(id)` is denied" in recipe + # Controls: only a runtime Responses row; Converse / Messages / Chat refused; absent → no flag. + assert "only when a `--target` row is a runtime GPT CRIS id on `responses`" in recipe + assert "`reason: stored_response_ops_no_responses_row` for a Converse / Anthropic Messages / Chat Completions plan" in recipe + assert "When the field is absent or `[]`, omit the flag" in recipe + + +def test_report_checklist_has_a_create_plus_retrieve_permission_check() -> None: + apply = _section(REPORT_GENERATOR.read_text(), "## How to Apply", "## How to Undo") + joined = " ".join(apply.split()) + assert "Create-plus-retrieve permission check" in joined + assert "Render ONLY when `analysis.stored_response_ops` is non-empty" in joined + assert "client.responses.create(" in joined and "client.responses.retrieve(r.id)" in joined + assert "the preflight ping only exercises creation" in joined + assert "`BedrockStoredResponseOps` statement" in joined and "project/default" in joined + + +def test_setup_contracts_name_the_get_invoke_grant_and_its_controls() -> None: + for path in (CANONICAL_GUIDE, GCP_GENERATE_AI, GCP_GENERATE_ARTIFACTS, AZURE_GENERATE_ARTIFACTS): + text = " ".join(path.read_text().split()) + assert "bedrock:GetInvoke" in text, f"{path}: no GetInvoke grant" + assert "client.responses.retrieve(id)" in text, path + assert "iam_policy.py --stored-response-ops retrieve[,cancel,delete]" in text, path + assert "read 2026-11-08" in text, f"{path}: record the read date" + # Only the retained ops; the grant stays on the default project ARN. + assert re.search(r"only (for )?the (ops actually )?retained", text), path + assert "project/default" in text, path + # Converse / Messages control. + assert re.search(r"`converse`(,| /) `messages`", text), f"{path}: converse / messages must be excluded" + # Setup verification does create-plus-retrieve. + for path in (GCP_GENERATE_ARTIFACTS, AZURE_GENERATE_ARTIFACTS, CANONICAL_GUIDE): + text = " ".join(path.read_text().split()) + assert "client.responses.retrieve(r.id)" in text, f"{path}: no create-plus-retrieve verification" + + +def test_iam_helper_grants_get_invoke_for_responses_only_and_refuses_converse_and_messages() -> None: + iam = _load("iam_policy") + sol, claude, nova = "us.openai.gpt-5.6-sol", "global.anthropic.claude-sonnet-5", "us.amazon.nova-lite-v1:0" + project = iam.default_project_arn("us-east-1", "123456789012") + pol = iam.generate_policy([claude, sol], "us-east-1", "123456789012", migration_path="runtime_messages", + target_apis={claude: "messages", sol: "responses"}, stored_response_ops=["retrieve"]) + stmt = {s["Sid"]: s for s in pol["Statement"]}["BedrockStoredResponseOps"] + assert stmt["Action"] == ["bedrock:GetInvoke"] and stmt["Resource"] == [project] + for apis in ({claude: "messages"}, {nova: "converse"}, {sol: "converse"}, {sol: "chat_completions"}): + try: + iam.generate_policy(list(apis), "us-east-1", "123456789012", target_apis=apis, + stored_response_ops=["retrieve"]) + except ValueError as e: + assert "stored-response ops" in str(e) + else: # pragma: no cover - the assertion above is the contract + raise AssertionError(f"GetInvoke granted for {apis}") + plain = iam.generate_policy([sol], "us-east-1", "123456789012", target_apis={sol: "responses"}) + assert "BedrockStoredResponseOps" not in [s["Sid"] for s in plain["Statement"]] + + +def test_vendored_openai_guides_match_canonical() -> None: + for vendored in VENDORED_GUIDES: + assert vendored.read_bytes() == CANONICAL_GUIDE.read_bytes(), vendored + + +# --- Finding 3: saved API reused only after the identity check --- + +def test_b4_checks_saved_identity_before_reusing_the_answer() -> None: + text = SKILL_MD.read_text() + b4 = " ".join(_section(text, "### B4 — Bedrock preflight", "## Phase C — Execute").split()) + assert "check the saved answer's identity BEFORE reusing it" in b4 + assert "--resume-runtime-api $PHASE_DIR/run-context.json --design $MIGRATION_DIR/aws-design-ai.json" in b4 + assert "`reason: saved_identity_match`" in b4 + assert "saved `assess_design_sha256` equals the current design's sha256" in b4 + assert "saved `target_models`" in b4 and "equal the current `bedrock_models[]` rows" in b4 + assert "`saved_identity_mismatch`" in b4 + assert "do NOT read the saved value" in b4 and "before any probe" in b4 + assert "A cached `messages` answer must never send a new GPT row to the Anthropic endpoint" in b4 + assert "a stale `responses` answer must never override a Chat Completions source" in b4 + # The old unconditional reuse sentence is gone from both places that had it. + assert "and reuse it without asking; otherwise AskUserQuestion" not in b4 + context = " ".join(_section(text, "### The context block", "### C0 — Run-context gate").split()) + assert "read `resolved_runtime_api` from run-context before asking again" not in context + assert "only when B4's `--resume-runtime-api` gate reports `saved_identity_match`" in context + + +def test_actual_gate_forces_re_resolution_on_identity_mismatch_and_reuses_on_match() -> None: + preflight = _load("preflight_bedrock") + gpt_rows = [{"source_model_id": "gpt-5.6-sol", "aws_model_id": "us.openai.gpt-5.6-sol"}] + saved = {"resolved_runtime_api": "messages", "assess_design_sha256": "a" * 64, + "target_models": [{"source_model": "claude-sonnet-4-5", + "aws_model_id": "global.anthropic.claude-sonnet-5", "use_case": "chat"}]} + # New GPT row under a changed design: the cached `messages` must not be reused. + mismatch = preflight.resume_runtime_api(saved, "b" * 64, gpt_rows) + assert mismatch["ok"] is False and mismatch["reason"] == "saved_identity_mismatch" + assert "resolved_runtime_api" not in mismatch + # Same design, same rows: reuse without re-asking. + saved_gpt = {"resolved_runtime_api": "chat_completions", "assess_design_sha256": "a" * 64, + "target_models": [{"source_model": "gpt-5.6-sol", "aws_model_id": "us.openai.gpt-5.6-sol", + "use_case": "chat"}]} + match = preflight.resume_runtime_api(saved_gpt, "a" * 64, gpt_rows) + assert match == {"ok": True, "reason": "saved_identity_match", "resolved_runtime_api": "chat_completions", + "detail": "Saved design, source and target identity match; reuse without asking."} + # Only the design changed (stale `responses` over a Chat source): still re-resolve. + stale = preflight.resume_runtime_api({**saved_gpt, "resolved_runtime_api": "responses"}, "c" * 64, gpt_rows) + assert stale["reason"] == "saved_identity_mismatch" From d955896ab6a2b62c83a556e7003d89277d495b8b Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 13:31:45 -0800 Subject: [PATCH 16/41] feat(aws-startup-advisor): detect Mantle-only selectors in Discover and record them as mantle_signals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discover now records the five code-detectable bedrock-mantle selectors (background_inference, web_search, server_side_tools, projects, workspaces) with file:line evidence so Clarify can confirm them instead of guessing the endpoint from source code alone. mantle_only_model_or_api stays Design-derived and in_region_processing is never detected (always asked). - skills/shared/ai/bedrock-endpoint-policy.md: new "## How a selector is learned" section (positive/negative constructs per selector, cites inference-responses-api.html read 2026-11-07); vendored copies re-synced (gcp-to-aws, azure-to-aws references/vendored/ai). - gcp-to-aws discover-app-code.md: Step 5C "Mantle-Only Selector Signals" (table + workloads[].mantle_signals[] output), Step 5B example, checklist. - azure-to-aws discover-app-code.md: Step 5C (bullet style), checklist. - gcp/azure schema-discover-ai.md: workloads[].mantle_signals[] shape, CRITICAL field bullet / enum, validation rule. - discover-openai-api.md / discover-anthropic-api.md / discover-openrouter-api.md (gcp + azure): Step 4 merge adds a projects / workspaces / web_search (:online) signal with the /line 0 sentinel; OpenRouter workspace_id is explicitly NOT a workspaces signal. - bedrock-migration-assessment discover-app-code.md: Step 4.5 with top-level mantle_signals[] output; Step 5 example and closing rule allow the key. - scripts/contracts/ai-workload-profile.schema.json (+ byte-identical BMA copy): optional root mantle_signals[] and mantleSignal definition. - validate-ai-profile.py: _check_mantle_signals (unmatched_mantle_signal) and mantle_signals_detected in the summary. - fixtures/bma-scan-openai-mantle-signals: positives on app/llm.py:5,9,14,21; negatives (function tool named web_search, store=True) on lines 30 and 37. - tests/test_mantle_selector_discovery.py (15 tests); test_bedrock_migration_assessment_core.py summary key set updated. Verification (plugins/aws-startup-advisor): - python3 tools/sync-vendored.py && --check: PASS (64 files, 2 synced) - python3 scripts/validate-artifacts.py --fixtures: PASS, 100 artifacts, 0 errors, no new baseline entries - python3 tools/check-skill-refs.py --check: PASS (1525 refs) - markdownlint-cli2 (823 tracked files): 0 errors - uv run --no-project --with pytest python -m pytest tests -q: 1456 passed, 222 subtests passed, 1 failed — the one failure is test_model_id_lint::test_real_plugin_passes flagging a retired model id inside the gitignored, untracked skills/llm-to-bedrock/scripts/.venv/ (anthropic SDK constants); pre-existing local environment, not a tracked file. Acceptance subset (6 files): 253 passed. - python3 tools/validate.py (repo root): All validations passed - git diff --check: clean --- .../ai-workload-profile.json | 22 +++ .../repo/app/llm.py | 38 ++++ .../repo/requirements.txt | 1 + .../contracts/ai-workload-profile.schema.json | 20 +- .../phases/discover/discover-anthropic-api.md | 8 + .../phases/discover/discover-app-code.md | 37 ++++ .../phases/discover/discover-openai-api.md | 8 + .../discover/discover-openrouter-api.md | 8 + .../references/shared/schema-discover-ai.md | 12 +- .../vendored/ai/bedrock-endpoint-policy.md | 46 +++++ .../contracts/ai-workload-profile.schema.json | 20 +- .../phases/discover/discover-app-code.md | 49 ++++- .../scripts/validate-ai-profile.py | 28 ++- .../phases/discover/discover-anthropic-api.md | 8 + .../phases/discover/discover-app-code.md | 44 +++++ .../phases/discover/discover-openai-api.md | 8 + .../discover/discover-openrouter-api.md | 8 + .../references/shared/schema-discover-ai.md | 16 ++ .../vendored/ai/bedrock-endpoint-policy.md | 46 +++++ .../shared/ai/bedrock-endpoint-policy.md | 46 +++++ .../test_bedrock_migration_assessment_core.py | 2 +- .../tests/test_mantle_selector_discovery.py | 185 ++++++++++++++++++ 22 files changed, 652 insertions(+), 8 deletions(-) create mode 100644 plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/ai-workload-profile.json create mode 100644 plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/repo/app/llm.py create mode 100644 plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/repo/requirements.txt create mode 100644 plugins/aws-startup-advisor/tests/test_mantle_selector_discovery.py diff --git a/plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/ai-workload-profile.json b/plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/ai-workload-profile.json new file mode 100644 index 000000000..b4d19af81 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/ai-workload-profile.json @@ -0,0 +1,22 @@ +{ + "schema_version": "1.0", + "ai_sources": [ + { + "provider": "openai", + "source_service": "openai_api", + "model_families": ["gpt"], + "model_evidence": "observed", + "models": ["gpt-4o"], + "model_observations": [ + {"normalized_model": "gpt-4o", "raw_model": "gpt-4o", "model_family": "gpt", "evidence": [{"provenance": "code", "confidence": "high", "detail": "app/llm.py:9 client.responses.create"}]} + ] + } + ], + "gateways": [], + "mantle_signals": [ + {"selector": "background_inference", "provider": "openai", "source_service": "openai_api", "model": "gpt-4o", "evidence": [{"provenance": "code", "confidence": "high", "detail": "app/llm.py:9 background=True"}]}, + {"selector": "web_search", "provider": "openai", "source_service": "openai_api", "model": "gpt-4o", "evidence": [{"provenance": "code", "confidence": "high", "detail": "app/llm.py:14 \"type\": \"web_search\""}]}, + {"selector": "server_side_tools", "provider": "openai", "source_service": "openai_api", "model": "gpt-4o", "evidence": [{"provenance": "code", "confidence": "high", "detail": "app/llm.py:21 \"type\": \"mcp\""}]}, + {"selector": "projects", "provider": "openai", "source_service": "openai_api", "model": "gpt-4o", "evidence": [{"provenance": "code", "confidence": "high", "detail": "app/llm.py:5 OpenAI-Project"}]} + ] +} diff --git a/plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/repo/app/llm.py b/plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/repo/app/llm.py new file mode 100644 index 000000000..c404d551d --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/repo/app/llm.py @@ -0,0 +1,38 @@ +import os + +from openai import OpenAI + +client = OpenAI(default_headers={"OpenAI-Project": os.environ.get("OPENAI_PROJECT_ID", "")}) + + +def start_report(text: str) -> str: + resp = client.responses.create(model="gpt-4o", background=True, input=f"Write a report on: {text}") + return resp.id + + +def research(question: str) -> str: + resp = client.responses.create(model="gpt-4o", tools=[{"type": "web_search"}], input=question) + return resp.output_text + + +def ask_docs(question: str) -> str: + resp = client.responses.create( + model="gpt-4o", + tools=[{"type": "mcp", "server_label": "docs", "server_url": "https://example.com/mcp"}], + input=question, + ) + return resp.output_text + + +def lookup(query: str) -> str: + resp = client.chat.completions.create( + model="gpt-4o", + tools=[{"type": "function", "function": {"name": "web_search", "parameters": {"type": "object"}}}], + messages=[{"role": "user", "content": query}], + ) + return resp.choices[0].message.content + + +def remember(note: str) -> str: + resp = client.responses.create(model="gpt-4o", store=True, input=note) + return resp.id diff --git a/plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/repo/requirements.txt b/plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/repo/requirements.txt new file mode 100644 index 000000000..eb1435f1b --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/repo/requirements.txt @@ -0,0 +1 @@ +openai>=1.0 diff --git a/plugins/aws-startup-advisor/scripts/contracts/ai-workload-profile.schema.json b/plugins/aws-startup-advisor/scripts/contracts/ai-workload-profile.schema.json index 37f8f425c..b2d26ebe8 100644 --- a/plugins/aws-startup-advisor/scripts/contracts/ai-workload-profile.schema.json +++ b/plugins/aws-startup-advisor/scripts/contracts/ai-workload-profile.schema.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "urn:awslabs:startups:migration:ai-workload-profile", "title": "AI workload profile (provider-neutral observation contract)", - "description": "Provider-neutral ai-workload-profile.json observation contract. bedrock-migration-assessment consumes it; no skill produces it yet. Structural validity does NOT imply an implemented detector, adapter, route, model mapping, or runtime-supported identity.", + "description": "Provider-neutral ai-workload-profile.json observation contract. bedrock-migration-assessment consumes it; no skill produces it yet. The optional mantle_signals[] array records the code-detectable Mantle-only selectors (background_inference, web_search, server_side_tools, projects, workspaces) with evidence; it is valid at every schema_version. Structural validity does NOT imply an implemented detector, adapter, route, model mapping, or runtime-supported identity.", "type": "object", "required": ["schema_version", "ai_sources", "gateways"], "additionalProperties": false, @@ -33,9 +33,27 @@ "type": "array", "uniqueItems": true, "items": {"$ref": "#/definitions/tool"} + }, + "mantle_signals": { + "description": "Code-detectable Mantle-only selectors observed by Discover, one entry per (selector, provider, source_service, model). Each (provider, source_service, model) must be an observed model of ai_sources[]; the validator enforces that. mantle_only_model_or_api and in_region_processing are never signals.", + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/mantleSignal"} } }, "definitions": { + "mantleSignal": { + "type": "object", + "required": ["selector", "provider", "source_service", "model", "evidence"], + "additionalProperties": false, + "properties": { + "selector": {"enum": ["background_inference", "web_search", "server_side_tools", "projects", "workspaces"]}, + "provider": {"$ref": "#/definitions/extensibleIdentity"}, + "source_service": {"$ref": "#/definitions/extensibleIdentity"}, + "model": {"$ref": "#/definitions/normalizedModel"}, + "evidence": {"type": "array", "minItems": 1, "items": {"$ref": "#/definitions/evidenceEntry"}} + } + }, "extensibleIdentity": {"type": "string", "pattern": "^(?!(?:both|openrouter|unknown)$)[a-z][a-z0-9_]*$"}, "modelFamily": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z][a-z0-9_.-]*$"}, "normalizedModel": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z0-9][a-z0-9._:/-]*$"}, diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-anthropic-api.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-anthropic-api.md index 77e4e8f33..a3c4841a0 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-anthropic-api.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-anthropic-api.md @@ -530,6 +530,14 @@ volume: already has this exact `workload_id` (the sha256 is deterministic per model, so a second merge of the same model naturally collides on it instead of duplicating). + c. **Mantle signal (`workspaces`):** when `$WORKSPACE_IDS` (Step 2) names + more than one workspace, or a workspace the user identified as + non-default, append `{ "selector": "workspaces", "file": "", + "line": 0, "construct": "anthropic-usage-profile.json workspace_ids: + " }` to `mantle_signals` of every Anthropic workload row in + `workloads[]` (existing or usage-only; create the array if absent, skip + an identical entry). Workspace ids only — never a key value. Clarify + confirms the signal (`schema-discover-ai.md` § workloads[]). Recompute `summary.total_models_detected` to include any (a) addition. 5. If `summary.ai_source` is `"azure_openai"` or `"openai"` and Anthropic diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-app-code.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-app-code.md index 9e94afb23..ffff9ecc6 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-app-code.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-app-code.md @@ -170,6 +170,40 @@ assign capability, collapse by `(model_id, sdk_method, structured_output)` into structured_output else "plain"))[:6]`. Azure OpenAI apps call the SAME `openai.*` SDK methods, so the capability map covers them unchanged. Every `workloads[].model_id` must appear in `models[]`. +## Step 5C: Mantle-only selector signals (ports from gcp) + +Five provider capabilities that decide the target endpoint later (`references/vendored/ai/bedrock-endpoint-policy.md`, +§ How a selector is learned) are visible in code. For each call site grouped in Step 5B, scan the +call arguments, the client construction it uses, and request headers. Azure OpenAI apps use the same +`openai.*` SDK, so the OpenAI constructs apply unchanged. Active (non-commented) code only. + +- `background_inference` — `background=True` / `background: true` on `responses.create`, or a raw + `"background": true` Responses body. NOT: `store=True`, `stream=True`, polling `responses.retrieve` + without `background`. +- `web_search` — `tools=[{"type": "web_search"}]`; `"type": "web_search_preview"` (any dated + `web_search_preview_*` form); Anthropic `"type": "web_search_20250305"`; OpenRouter + `plugins: [{"id": "web"}]` or a `:online` model suffix. NOT: a client-side `"type": "function"` tool + merely named `web_search`. +- `server_side_tools` — a tools entry of `"type": "mcp"` (`server_url` / `server_label`), + `code_interpreter`, `file_search`, `computer_use_preview`, or `image_generation` used as a tool type; + Anthropic `code_execution_*` / `text_editor_*` / `computer_*` / `bash_*` tool types — any + provider-executed tool other than web search. NOT: `"type": "function"` tools, Gemini + `function_declarations=`, `@tool` decorators, `tool_choice` — client-side function tools never count. +- `projects` — `OpenAI-Project` header (`default_headers={"OpenAI-Project": ...}`), `project=` kwarg + on the OpenAI client, or an `OPENAI_PROJECT_ID`-style env name read in code. NOT: + `OpenAI-Organization` alone; no header means the default project. +- `workspaces` — `anthropic-workspace-id` header or an `ANTHROPIC_WORKSPACE_ID`-style env name. NOT: + OpenRouter's own `workspace_id` scoping. + +`mantle_only_model_or_api` is not a scanner output (Design derives it from the catalog) and +`in_region_processing` is never code-detectable (Clarify always asks it). Never write either value. + +**Output:** append to that workload's entry `"mantle_signals": [{ "selector": "web_search", +"file": "app/llm.py", "line": 14, "construct": "tools=[{\"type\": \"web_search\"}]" }]` — one entry +per `(selector, call site)`; repo-relative POSIX `file`; `construct` is the matched source text (an env +var NAME is fine, its value never is; no prompt or comment text). Omit the key or write `[]` when +nothing matched. These are evidence for Clarify to confirm, not a decision; name no AWS endpoint here. + ## Step 5.5: Agentic extraction (only if `is_agentic: true`) — ports unchanged Per agent: `agent_id` (snake-cased), `file`, `line`, `model_id`, `tools[]`, `memory_type` @@ -231,6 +265,9 @@ The assembler (`discover-assemble.md`) writes the file; do not update `.phase-st - [ ] Every `models[]` entry has the exact-named fields; `detected_via` lists only analyzed sources. - [ ] `workloads[]` present; every `workloads[].model_id` ∈ `models[].model_id`; `workload_id` matches `wl_[0-9a-f]{6}`. +- [ ] `workloads[].mantle_signals` (optional): `selector` ∈ + `background_inference | web_search | server_side_tools | projects | workspaces`; cited lines + are real and contain the `construct`; no secret values. - [ ] `integration.gateway_type` set (or null); `capabilities_summary` consistent with `models[].capabilities_used`. - [ ] `is_agentic` cases: `agentic_profile`+`tool_manifest` present iff true; `agent_count` == diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-openai-api.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-openai-api.md index 5b9ab1e0b..8a9db9ca4 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-openai-api.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-openai-api.md @@ -398,6 +398,14 @@ and volume: `workloads[]` entry already has this exact `workload_id` (the sha256 is deterministic per model, so a second merge of the same model naturally collides on it instead of duplicating). + c. **Mantle signal (`projects`):** when `$PROJECT_IDS` (the `project_filter` + written in Step 2) names more than one project, or a project the user + identified as non-default, append `{ "selector": "projects", "file": + "", "line": 0, "construct": "openai-usage-profile.json + project_filter: " }` to `mantle_signals` of every direct-OpenAI + workload row in `workloads[]` (existing or usage-only; create the array + if absent, skip an identical entry). Project ids only — never a key + value. Clarify confirms the signal (`schema-discover-ai.md` § workloads[]). Recompute `summary.total_models_detected` to include any (a) addition. 5. If `summary.ai_source` is `"azure_openai"` and direct OpenAI usage was diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-openrouter-api.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-openrouter-api.md index ddfb52c2b..85e438e09 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-openrouter-api.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-openrouter-api.md @@ -795,6 +795,14 @@ volume: existing `workloads[]` entry already has this exact `workload_id` (the sha256 is deterministic per model, so a second merge of the same model naturally collides on it instead of duplicating). + c. **Mantle signal (`web_search`):** for any `usage_by_model` model string + carrying the `:online` suffix (OpenRouter's web-search plugin shorthand), + append `{ "selector": "web_search", "file": "", "line": 0, + "construct": "openrouter-usage-profile.json model: " }` to + `mantle_signals` of that model's workload row (create the array if + absent, skip an identical entry). OpenRouter's `workspace_id` scoping + (Step 1.5) is OpenRouter's own account concept and is NOT a `workspaces` + signal. Clarify confirms the signal (`schema-discover-ai.md` § workloads[]). Recompute `summary.total_models_detected` to include any (a) addition (it counts `models[]` length, not just code-derived models). diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-discover-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-discover-ai.md index 5cb61f224..b3b18a927 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-discover-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-discover-ai.md @@ -208,13 +208,21 @@ after Clarify confirms them**, not this array. "capability": "text_generation", // see enum below "capability_confidence": "high", // high | medium | low "structured_output": false, - "call_sites": [{ "file": "app/llm.py", "line": 42 }] // non-empty, POSIX repo-relative + "call_sites": [{ "file": "app/llm.py", "line": 42 }], // non-empty, POSIX repo-relative + "mantle_signals": [{ "selector": "projects", "file": "app/llm.py", "line": 9, "construct": "default_headers={\"OpenAI-Project\": os.environ[\"OPENAI_PROJECT_ID\"]}" }] // optional — discover-app-code.md Step 5C } ``` `capability` enum: `text_generation`, `structured_output`, `image_generation`, `embedding`, `speech_to_text`, `text_to_speech`, `document_extraction`, `image_analysis`, `speech_transcription`, -`unknown`. The capability map is `references/vendored/ai/sdk-capability-map.json` (keys on SDK call +`unknown`. +`mantle_signals[].selector` enum: `background_inference`, `web_search`, `server_side_tools`, +`projects`, `workspaces` — the only code-detectable Mantle selectors; never +`mantle_only_model_or_api` (Design derives it) or `in_region_processing` (Clarify always asks). +Written by `discover-app-code.md` Step 5C with a real file:line and the matched source text as +`construct` (never a secret value), and by the usage-API Step 4 merges (`discover-openai-api.md` → +`projects`, `discover-anthropic-api.md` → `workspaces`, `discover-openrouter-api.md` → `web_search`) +with the `""` / `line: 0` sentinel. Omit or `[]` when nothing matched. The capability map is `references/vendored/ai/sdk-capability-map.json` (keys on SDK call pattern, so Azure-OpenAI-via-openai-SDK is already covered). Azure Cognitive SDK methods (`azure.ai.vision.*`, `azure.ai.formrecognizer.*`, `azure.cognitiveservices.speech.*`, `azure.ai.textanalytics.*`) map to `image_analysis` / `document_extraction` / `speech_transcription` diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md index fdc4948a2..ac1d3ca94 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md @@ -51,6 +51,52 @@ defect: rewrite the row as the `bedrock-runtime` path (`runtime_openai_cris` wit app's `required_client_api`, or `runtime_messages`) or justify it with one of the values above. A `runtime_*` or `converse` row never carries `mantle_selector`. +## How a selector is learned + +The seven values are learned in three different ways; no phase may guess one it cannot learn. + +- **Detected in code, confirmed by the user.** `background_inference`, `web_search`, + `server_side_tools`, `projects` and `workspaces` are the only code-detectable selectors. + Discover's app-code scanners record each match with file:line evidence — as + `workloads[].mantle_signals[]` in the gcp-to-aws / azure-to-aws profile + (`discover-app-code.md` Step 5C) and as top-level `mantle_signals[]` in the provider-neutral + profile (bedrock-migration-assessment `discover-app-code.md` Step 4.5); the usage-API discovery + files add `projects` / `workspaces` / `web_search` signals from the account's own scoping and + model strings. Clarify shows the detected signals and the user confirms or drops each one into + `ai_constraints.mantle_requirements`; when no code was scanned, Clarify asks the same five + outright. A detected signal is a prompt for the question, never the answer. + - `background_inference` — positive: `background=True` / `background: true` on + `responses.create`, or a raw `"background": true` Responses body. Not a signal: `store=True`, + `stream=True`, or polling `responses.retrieve` without `background`. `bedrock-runtime` + rejects `background=true` with a 400 + (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, read + 2026-11-07). + - `web_search` — positive: `tools=[{"type": "web_search"}]`, `"type": "web_search_preview"` + (any dated `web_search_preview_*` form), Anthropic `"type": "web_search_20250305"`, + OpenRouter `plugins: [{"id": "web"}]` or a `:online` model suffix. Not a signal: a + client-side `"type": "function"` tool merely named `web_search`. + - `server_side_tools` — positive: a tools entry of `"type": "mcp"` (`server_url` / + `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, + `image_generation` used as a tool type, or an Anthropic `code_execution_*` / `text_editor_*` + / `computer_*` / `bash_*` tool type — any provider-executed tool other than web search. Not a + signal: `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, + `tool_choice` — client-side function tools never count. + - `projects` — positive: an `OpenAI-Project` header + (`default_headers={"OpenAI-Project": ...}`), a `project=` kwarg on the OpenAI client, or an + `OPENAI_PROJECT_ID`-style environment name read in code; in usage discovery, a + `project_filter` naming more than one project or a non-default project. Not a signal: + `OpenAI-Organization` alone; no header means the default project. + - `workspaces` — positive: an `anthropic-workspace-id` header or an `ANTHROPIC_WORKSPACE_ID`-style + environment name; in usage discovery, `workspace_ids` naming more than one workspace or a + non-default workspace. Not a signal: OpenRouter's own `workspace_id` scoping, which is an + OpenRouter account concept. +- **Derived by Design, never asked.** `mantle_only_model_or_api` is derived from the model id and + the required API against the access-path tables in `ai-openai-to-bedrock.md` / + `openai-on-bedrock.md`; it is not a scanner output and not a Clarify question. +- **Always asked, never detected.** `in_region_processing` is never code-detectable: nothing in a + call site says whether cross-Region inference is acceptable. Clarify always asks it together + with the Region. + ## Public comparison (read 2026-11-07) | Item | `bedrock-runtime` | `bedrock-mantle` | Source page | diff --git a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/contracts/ai-workload-profile.schema.json b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/contracts/ai-workload-profile.schema.json index 37f8f425c..b2d26ebe8 100644 --- a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/contracts/ai-workload-profile.schema.json +++ b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/contracts/ai-workload-profile.schema.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "urn:awslabs:startups:migration:ai-workload-profile", "title": "AI workload profile (provider-neutral observation contract)", - "description": "Provider-neutral ai-workload-profile.json observation contract. bedrock-migration-assessment consumes it; no skill produces it yet. Structural validity does NOT imply an implemented detector, adapter, route, model mapping, or runtime-supported identity.", + "description": "Provider-neutral ai-workload-profile.json observation contract. bedrock-migration-assessment consumes it; no skill produces it yet. The optional mantle_signals[] array records the code-detectable Mantle-only selectors (background_inference, web_search, server_side_tools, projects, workspaces) with evidence; it is valid at every schema_version. Structural validity does NOT imply an implemented detector, adapter, route, model mapping, or runtime-supported identity.", "type": "object", "required": ["schema_version", "ai_sources", "gateways"], "additionalProperties": false, @@ -33,9 +33,27 @@ "type": "array", "uniqueItems": true, "items": {"$ref": "#/definitions/tool"} + }, + "mantle_signals": { + "description": "Code-detectable Mantle-only selectors observed by Discover, one entry per (selector, provider, source_service, model). Each (provider, source_service, model) must be an observed model of ai_sources[]; the validator enforces that. mantle_only_model_or_api and in_region_processing are never signals.", + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/definitions/mantleSignal"} } }, "definitions": { + "mantleSignal": { + "type": "object", + "required": ["selector", "provider", "source_service", "model", "evidence"], + "additionalProperties": false, + "properties": { + "selector": {"enum": ["background_inference", "web_search", "server_side_tools", "projects", "workspaces"]}, + "provider": {"$ref": "#/definitions/extensibleIdentity"}, + "source_service": {"$ref": "#/definitions/extensibleIdentity"}, + "model": {"$ref": "#/definitions/normalizedModel"}, + "evidence": {"type": "array", "minItems": 1, "items": {"$ref": "#/definitions/evidenceEntry"}} + } + }, "extensibleIdentity": {"type": "string", "pattern": "^(?!(?:both|openrouter|unknown)$)[a-z][a-z0-9_]*$"}, "modelFamily": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z][a-z0-9_.-]*$"}, "normalizedModel": {"type": "string", "pattern": "^(?!(?:both|unknown)$)[a-z0-9][a-z0-9._:/-]*$"}, diff --git a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/discover/discover-app-code.md b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/discover/discover-app-code.md index 4e1d50b38..8e58dc8d8 100644 --- a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/discover/discover-app-code.md +++ b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/discover/discover-app-code.md @@ -192,6 +192,42 @@ When a source's SDK is used but no model string can be located (for example, it --- +## Step 4.5: Mantle-only selector signals + +A few provider capabilities decide the target endpoint later (`skills/shared/ai/bedrock-endpoint-policy.md`, § How a selector is learned). Five of them are visible in code. For every call site whose model was observed in Step 4, scan the call arguments, the client construction that the call site uses, and any request headers for the constructs below. Active (non-commented) code only. + +| Selector | Positive constructs (any one) | Does NOT count | +| ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `background_inference` | `background=True` / `background: true` on `responses.create`, or a raw `"background": true` key in a Responses request body | `store=True`, `stream=True`, polling `responses.retrieve` without `background` | +| `web_search` | `tools=[{"type": "web_search"}]`; `"type": "web_search_preview"` (any dated `web_search_preview_*` form); Anthropic `"type": "web_search_20250305"`; OpenRouter `plugins: [{"id": "web"}]` or a `:online` model suffix | A client-side `"type": "function"` tool merely NAMED `web_search` | +| `server_side_tools` | A tools entry of `"type": "mcp"` (`server_url` / `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, or `image_generation` used as a tool type; Anthropic `code_execution_*` / `text_editor_*` / `computer_*` / `bash_*` tool types — any provider-executed tool other than web search | `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, `tool_choice` — client-side function tools never count | +| `projects` | `OpenAI-Project` header (`default_headers={"OpenAI-Project": ...}`), `project=` kwarg on the OpenAI client, or an `OPENAI_PROJECT_ID`-style environment variable name read in code | `OpenAI-Organization` alone; no header means the default project | +| `workspaces` | `anthropic-workspace-id` header or an `ANTHROPIC_WORKSPACE_ID`-style environment variable name | OpenRouter's own `workspace_id` scoping (an OpenRouter account concept) | + +Two selectors are NOT scanner outputs: `mantle_only_model_or_api` is derived by Design from the model and API against the catalog, and `in_region_processing` is never code-detectable — Clarify always asks it. Never write either value into `mantle_signals`. + +**Output:** a top-level `mantle_signals` array, one entry per `(selector, provider, source_service, model)`: + +```json +"mantle_signals": [ + { + "selector": "web_search", + "provider": "openai", + "source_service": "openai_api", + "model": "gpt-4o", + "evidence": [{"provenance": "code", "confidence": "high", "detail": "app/llm.py:14 tools type web_search"}] + } +] +``` + +- `(provider, source_service, model)` must be an observed model of an `ai_sources[]` entry; the checker rejects a signal that names anything else. +- `evidence[].detail` is `path:line` plus the construct name, exactly as the Step 5 model evidence; never a secret value, prompt text, or comment text. An environment variable NAME is fine; its value is not. +- Omit the key when nothing matched. + +These signals are evidence for Clarify to confirm with the user, not a decision; name no AWS endpoint here. + +--- + ## Step 5: Write ai-workload-profile.json Write the profile to the path the Discover controller names: @@ -216,11 +252,20 @@ Write the profile to the path the Discover controller names: ] } ], - "gateways": [] + "gateways": [], + "mantle_signals": [ + { + "selector": "projects", + "provider": "openai", + "source_service": "openai_api", + "model": "gpt-4o", + "evidence": [{"provenance": "code", "confidence": "high", "detail": "app/llm.py:3 default_headers OpenAI-Project"}] + } + ] } ``` - One `model_observations` entry per entry in `models`, with every evidence entry using `provenance: "code"`. Keep `detail` to a file and line plus the call name; never copy secret values, prompts, or comment text into it. - `model_families` lists every family of the source's observed models. - For an observed gateway, each upstream lists exactly the families and models of the matching source; the checker rejects anything else. -- Write only `schema_version`, `ai_sources`, and `gateways`. +- Write only `schema_version`, `ai_sources`, `gateways`, and — when Step 4.5 matched anything — `mantle_signals` (omit the key when none). diff --git a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/scripts/validate-ai-profile.py b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/scripts/validate-ai-profile.py index d4306e902..02a66a6f7 100644 --- a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/scripts/validate-ai-profile.py +++ b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/scripts/validate-ai-profile.py @@ -212,6 +212,29 @@ def _check_v11(profile: dict, sources: dict[tuple[str, str], dict]) -> None: raise ProfileRejected("unmatched_tool_agent", f"tool {tool['name']} names unknown agents {unknown}") +def _check_mantle_signals(profile: dict, sources: dict[tuple[str, str], dict]) -> None: + for signal in profile.get("mantle_signals", []): + key = (signal["provider"], signal["source_service"]) + if key not in sources or signal["model"] not in sources[key]["models"]: + raise ProfileRejected( + "unmatched_mantle_signal", + f"{signal['selector']}: {key[0]}/{key[1]} {signal['model']} is not an observed model", + ) + + +def _mantle_signals_detected(profile: dict) -> list[dict]: + return [ + { + "selector": signal["selector"], + "provider": signal["provider"], + "source_service": signal["source_service"], + "model": signal["model"], + "detail": signal["evidence"][0].get("detail"), + } + for signal in profile.get("mantle_signals", []) + ] + + def _framework_detected(profile: dict) -> list[str] | None: integration = profile.get("integration") if integration is None or "agentic_profile" in profile: @@ -279,6 +302,7 @@ def _summary(profile: dict) -> dict: "models_extractable": all_high, "ai_framework_detected": _framework_detected(profile), "ai_vision_detected": _input_types_detected(profile), + "mantle_signals_detected": _mantle_signals_detected(profile), } @@ -302,7 +326,9 @@ def validate_profile(profile: Any, schema: dict | None = None) -> dict: errors = _schema_errors(schema, profile, schema) if errors: raise ProfileRejected("schema_violation", "; ".join(errors[:5])) - _check_v11(profile, _check_semantics(profile)) + sources = _check_semantics(profile) + _check_v11(profile, sources) + _check_mantle_signals(profile, sources) return _summary(profile) diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-anthropic-api.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-anthropic-api.md index 89ead4c1a..933750cfc 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-anthropic-api.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-anthropic-api.md @@ -530,6 +530,14 @@ volume: already has this exact `workload_id` (the sha256 is deterministic per model, so a second merge of the same model naturally collides on it instead of duplicating). + c. **Mantle signal (`workspaces`):** when `$WORKSPACE_IDS` (Step 2) names + more than one workspace, or a workspace the user identified as + non-default, append `{ "selector": "workspaces", "file": "", + "line": 0, "construct": "anthropic-usage-profile.json workspace_ids: + " }` to `mantle_signals` of every Anthropic workload row in + `workloads[]` (existing or usage-only; create the array if absent, skip + an identical entry). Workspace ids only — never a key value. Clarify + confirms the signal (`schema-discover-ai.md` § workloads[]). Recompute `summary.total_models_detected` to include any (a) addition. 5. If `summary.ai_source` is `"gemini"` or `"openai"` and Anthropic usage was diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-app-code.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-app-code.md index 761b5d38c..607e80ad9 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-app-code.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-app-code.md @@ -363,6 +363,20 @@ After extracting model details (Step 5), split the detected AI usage into distin "call_sites": [ { "file": "lib/imagen.ts", "line": 60 } ] + }, + { + "workload_id": "wl_9532e7", + "model_id": "gpt-4o", + "sdk_method": "openai.chat.completions.create", + "capability": "text_generation", + "capability_confidence": "medium", + "structured_output": false, + "call_sites": [ + { "file": "src/ai/openai_client.py", "line": 22 } + ], + "mantle_signals": [ + { "selector": "projects", "file": "src/ai/openai_client.py", "line": 9, "construct": "default_headers={\"OpenAI-Project\": os.environ[\"OPENAI_PROJECT_ID\"]}" } + ] } ] ``` @@ -374,6 +388,35 @@ After extracting model details (Step 5), split the detected AI usage into distin - Every model_id in `workloads[]` MUST also appear in `models[]` (backward compatibility) - If a method is in the structured-output trio AND the model is multimodal (Gemini, GPT-4o, Claude) AND structured output is NOT detected → set `capability_confidence: "medium"` (ambiguous — Clarify will confirm) +## Step 5C: Mantle-Only Selector Signals + +A few provider capabilities decide the target endpoint later (`references/vendored/ai/bedrock-endpoint-policy.md`, § How a selector is learned). Five of them are visible in code. For each call site grouped into a workload in Step 5B, scan the call arguments, the client construction that the call site uses, and any request headers for the constructs below. Record a signal only for active (non-commented) code. + +| Selector | Positive constructs (any one) | Does NOT count | +| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `background_inference` | `background=True` / `background: true` on `responses.create`, or a raw `"background": true` key in a Responses request body | `store=True`, `stream=True`, polling `responses.retrieve` without `background` | +| `web_search` | `tools=[{"type": "web_search"}]`; `"type": "web_search_preview"` (any dated `web_search_preview_*` form); Anthropic `"type": "web_search_20250305"`; OpenRouter `plugins: [{"id": "web"}]` or a `:online` model suffix | A client-side `"type": "function"` tool merely NAMED `web_search` | +| `server_side_tools` | A tools entry of `"type": "mcp"` (`server_url` / `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, or `image_generation` used as a tool type; Anthropic `code_execution_*` / `text_editor_*` / `computer_*` / `bash_*` tool types — any provider-executed tool other than web search | `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, `tool_choice` — client-side function tools never count | +| `projects` | `OpenAI-Project` header (`default_headers={"OpenAI-Project": ...}`), `project=` kwarg on the OpenAI client, or an `OPENAI_PROJECT_ID`-style env name read in code | `OpenAI-Organization` alone; no header means the default project | +| `workspaces` | `anthropic-workspace-id` header or an `ANTHROPIC_WORKSPACE_ID`-style env name | OpenRouter's own `workspace_id` scoping (an OpenRouter account concept) | + +Two selectors are NOT scanner outputs: `mantle_only_model_or_api` is derived by Design from the model id and API against the catalog, and `in_region_processing` is never code-detectable — Clarify always asks it. Never write either value into `mantle_signals`. + +**Output:** append a `mantle_signals` array to that workload's entry in `workloads[]`, one entry per `(selector, call site)`: + +```json +"mantle_signals": [ + { "selector": "web_search", "file": "src/ai/openai_client.py", "line": 14, "construct": "tools=[{\"type\": \"web_search\"}]" } +] +``` + +- `selector` is one of the five values in the table above. +- `file` is a repo-relative POSIX path; `line` is the line of the matched construct (the client construction line for `projects` / `workspaces`, the call line otherwise). +- `construct` is the matched source text — the argument or header as written, never a secret value, prompt text, or comment text. An env var NAME is fine; its value is not. +- Omit the key, or write `[]`, when nothing matched for that workload. + +These signals are evidence for Clarify, not a decision: Clarify asks the user to confirm each one, and Design chooses the endpoint. Do not name AWS endpoints here. + --- **Skip this step entirely if `is_agentic: false` from Step 3.5.** @@ -633,6 +676,7 @@ After generating the output file, the parent `discover.md` handles the phase sta - Every `workloads[].model_id` also appears in `models[].model_id` - `workloads[].capability_confidence` is one of: `"high"`, `"medium"`, `"low"` - `workloads[].workload_id` matches pattern `wl_[0-9a-f]{6}` +- `workloads[].mantle_signals` (optional, Step 5C): every entry has `selector` ∈ `background_inference | web_search | server_side_tools | projects | workspaces`, a repo-relative `file`, a real `line` that contains the quoted `construct`, and no secret value; never `mantle_only_model_or_api` or `in_region_processing` - If `is_agentic: true`: `agentic_profile` section exists with all required fields - If `is_agentic: true`: `agentic_profile.agent_count` equals length of `agentic_profile.agents[]` - If `is_agentic: true`: `agentic_profile.tool_count` equals length of `tool_manifest[]` (deduplicated) diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-openai-api.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-openai-api.md index b25248bb2..47d7ae1ae 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-openai-api.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-openai-api.md @@ -336,6 +336,14 @@ volume: OpenAI usage data — call sites not yet located in code" }`. Code-derived entries always win on conflict; usage-only entries tell Clarify what code analysis missed. + - **Mantle signal (`projects`):** when `$PROJECT_IDS` (the `project_filter` + written in Step 2) names more than one project, or a project the user + identified as non-default, append `{ "selector": "projects", "file": + "", "line": 0, "construct": "openai-usage-profile.json + project_filter: " }` to `mantle_signals` of every OpenAI workload + row in `workloads[]` (existing or usage-only; create the array if absent, + skip an identical entry). Project ids only — never a key value. Clarify + confirms the signal (`schema-discover-ai.md` § workloads[]). 5. If `summary.ai_source` is `"gemini"` and OpenAI usage was found, set it to `"both"`. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-openrouter-api.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-openrouter-api.md index c947ce5f7..93893caad 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-openrouter-api.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-openrouter-api.md @@ -794,6 +794,14 @@ volume: existing `workloads[]` entry already has this exact `workload_id` (the sha256 is deterministic per model, so a second merge of the same model naturally collides on it instead of duplicating). + c. **Mantle signal (`web_search`):** for any `usage_by_model` model string + carrying the `:online` suffix (OpenRouter's web-search plugin shorthand), + append `{ "selector": "web_search", "file": "", "line": 0, + "construct": "openrouter-usage-profile.json model: " }` to + `mantle_signals` of that model's workload row (create the array if + absent, skip an identical entry). OpenRouter's `workspace_id` scoping + (Step 1.5) is OpenRouter's own account concept and is NOT a `workspaces` + signal. Clarify confirms the signal (`schema-discover-ai.md` § workloads[]). Recompute `summary.total_models_detected` to include any (a) addition (it counts `models[]` length, not just code-derived models). diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-discover-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-discover-ai.md index 064e32858..cf213271c 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-discover-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-discover-ai.md @@ -212,6 +212,20 @@ Generated by `discover-app-code.md` Step 5B. Splits the detected AI usage into d "call_sites": [ { "file": "lib/gemini.ts", "line": 27 } ] + }, + { + "workload_id": "wl_9532e7", + "model_id": "gpt-4o", + "sdk_method": "openai.chat.completions.create", + "capability": "text_generation", + "capability_confidence": "medium", + "structured_output": false, + "call_sites": [ + { "file": "lib/openai.ts", "line": 14 } + ], + "mantle_signals": [ + { "selector": "web_search", "file": "lib/openai.ts", "line": 14, "construct": "tools: [{ type: 'web_search' }]" } + ] } ] } @@ -226,6 +240,7 @@ Generated by `discover-app-code.md` Step 5B. Splits the detected AI usage into d - `capability_confidence` — `"high"`, `"medium"`, or `"low"` (NOT `confidence`, `confidence_level`) - `structured_output` — Boolean: true iff structured-output indicators detected in call arguments - `call_sites` — Array of `{ "file": string, "line": integer }` — repo-relative POSIX paths +- `mantle_signals` — OPTIONAL array of `{ "selector": string, "file": string, "line": integer, "construct": string }` (NOT `mantle_selectors`, `signals`). `selector` ∈ `background_inference | web_search | server_side_tools | projects | workspaces` — the only code-detectable Mantle selectors. Produced by `discover-app-code.md` Step 5C (real file:line, `construct` = the matched source text, never a secret value) and by the usage-API Step 4 merges (`discover-openai-api.md` → `projects`, `discover-anthropic-api.md` → `workspaces`, `discover-openrouter-api.md` → `web_search`), which use the `""` / `line: 0` sentinel because they have no call site. Omit the key or write `[]` when nothing matched. Clarify confirms each signal with the user; Design reads the confirmed answer, not this array. **Capability assignment rules (from `references/vendored/ai/sdk-capability-map.json`):** @@ -244,6 +259,7 @@ Generated by `discover-app-code.md` Step 5B. Splits the detected AI usage into d - `workload_id` is unique within the array and deterministic across re-runs - `call_sites` is non-empty for every workload entry — EXCEPT a usage-only workload (see below), where the sentinel form satisfies "non-empty" without inventing a location that was never observed - `call_sites[].file` uses repo-relative POSIX paths (forward slashes, no absolute paths) — EXCEPT the usage-only sentinel `""`, which is not a real path and must not be treated as one +- Every `mantle_signals[].selector` is one of `background_inference`, `web_search`, `server_side_tools`, `projects`, `workspaces` — never `mantle_only_model_or_api` (Design derives it from the catalog) or `in_region_processing` (never code-detectable; Clarify always asks) **Usage-only workloads (a model billed via a usage API but never located in code — e.g. `discover-openrouter-api.md` Step 4).** A usage-only model has no call site to report; do NOT invent one (a fabricated `{file, line}` would misattribute the finding to code that doesn't call that model). Instead: `sdk_method: "usage_api"`, `structured_output: false`, `capability_confidence: "low"` (never located in code, so capability is a guess from usage volume alone — `"text_generation"` unless the usage API's own data names something more specific), and `call_sites: [{"file": "", "line": 0}]` — a fixed sentinel, not a real path, satisfying the non-empty-`call_sites` rule above without fabricating a location. Every consumer of `call_sites[].file` (there are none downstream today — Design reads only `workload_id`/`model_id`/`capability`/`capability_confidence`, per `design-ai.md`) must treat this sentinel as "no location," not as a literal file to open. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md index fdc4948a2..ac1d3ca94 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md @@ -51,6 +51,52 @@ defect: rewrite the row as the `bedrock-runtime` path (`runtime_openai_cris` wit app's `required_client_api`, or `runtime_messages`) or justify it with one of the values above. A `runtime_*` or `converse` row never carries `mantle_selector`. +## How a selector is learned + +The seven values are learned in three different ways; no phase may guess one it cannot learn. + +- **Detected in code, confirmed by the user.** `background_inference`, `web_search`, + `server_side_tools`, `projects` and `workspaces` are the only code-detectable selectors. + Discover's app-code scanners record each match with file:line evidence — as + `workloads[].mantle_signals[]` in the gcp-to-aws / azure-to-aws profile + (`discover-app-code.md` Step 5C) and as top-level `mantle_signals[]` in the provider-neutral + profile (bedrock-migration-assessment `discover-app-code.md` Step 4.5); the usage-API discovery + files add `projects` / `workspaces` / `web_search` signals from the account's own scoping and + model strings. Clarify shows the detected signals and the user confirms or drops each one into + `ai_constraints.mantle_requirements`; when no code was scanned, Clarify asks the same five + outright. A detected signal is a prompt for the question, never the answer. + - `background_inference` — positive: `background=True` / `background: true` on + `responses.create`, or a raw `"background": true` Responses body. Not a signal: `store=True`, + `stream=True`, or polling `responses.retrieve` without `background`. `bedrock-runtime` + rejects `background=true` with a 400 + (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, read + 2026-11-07). + - `web_search` — positive: `tools=[{"type": "web_search"}]`, `"type": "web_search_preview"` + (any dated `web_search_preview_*` form), Anthropic `"type": "web_search_20250305"`, + OpenRouter `plugins: [{"id": "web"}]` or a `:online` model suffix. Not a signal: a + client-side `"type": "function"` tool merely named `web_search`. + - `server_side_tools` — positive: a tools entry of `"type": "mcp"` (`server_url` / + `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, + `image_generation` used as a tool type, or an Anthropic `code_execution_*` / `text_editor_*` + / `computer_*` / `bash_*` tool type — any provider-executed tool other than web search. Not a + signal: `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, + `tool_choice` — client-side function tools never count. + - `projects` — positive: an `OpenAI-Project` header + (`default_headers={"OpenAI-Project": ...}`), a `project=` kwarg on the OpenAI client, or an + `OPENAI_PROJECT_ID`-style environment name read in code; in usage discovery, a + `project_filter` naming more than one project or a non-default project. Not a signal: + `OpenAI-Organization` alone; no header means the default project. + - `workspaces` — positive: an `anthropic-workspace-id` header or an `ANTHROPIC_WORKSPACE_ID`-style + environment name; in usage discovery, `workspace_ids` naming more than one workspace or a + non-default workspace. Not a signal: OpenRouter's own `workspace_id` scoping, which is an + OpenRouter account concept. +- **Derived by Design, never asked.** `mantle_only_model_or_api` is derived from the model id and + the required API against the access-path tables in `ai-openai-to-bedrock.md` / + `openai-on-bedrock.md`; it is not a scanner output and not a Clarify question. +- **Always asked, never detected.** `in_region_processing` is never code-detectable: nothing in a + call site says whether cross-Region inference is acceptable. Clarify always asks it together + with the Region. + ## Public comparison (read 2026-11-07) | Item | `bedrock-runtime` | `bedrock-mantle` | Source page | diff --git a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md index fdc4948a2..ac1d3ca94 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md @@ -51,6 +51,52 @@ defect: rewrite the row as the `bedrock-runtime` path (`runtime_openai_cris` wit app's `required_client_api`, or `runtime_messages`) or justify it with one of the values above. A `runtime_*` or `converse` row never carries `mantle_selector`. +## How a selector is learned + +The seven values are learned in three different ways; no phase may guess one it cannot learn. + +- **Detected in code, confirmed by the user.** `background_inference`, `web_search`, + `server_side_tools`, `projects` and `workspaces` are the only code-detectable selectors. + Discover's app-code scanners record each match with file:line evidence — as + `workloads[].mantle_signals[]` in the gcp-to-aws / azure-to-aws profile + (`discover-app-code.md` Step 5C) and as top-level `mantle_signals[]` in the provider-neutral + profile (bedrock-migration-assessment `discover-app-code.md` Step 4.5); the usage-API discovery + files add `projects` / `workspaces` / `web_search` signals from the account's own scoping and + model strings. Clarify shows the detected signals and the user confirms or drops each one into + `ai_constraints.mantle_requirements`; when no code was scanned, Clarify asks the same five + outright. A detected signal is a prompt for the question, never the answer. + - `background_inference` — positive: `background=True` / `background: true` on + `responses.create`, or a raw `"background": true` Responses body. Not a signal: `store=True`, + `stream=True`, or polling `responses.retrieve` without `background`. `bedrock-runtime` + rejects `background=true` with a 400 + (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, read + 2026-11-07). + - `web_search` — positive: `tools=[{"type": "web_search"}]`, `"type": "web_search_preview"` + (any dated `web_search_preview_*` form), Anthropic `"type": "web_search_20250305"`, + OpenRouter `plugins: [{"id": "web"}]` or a `:online` model suffix. Not a signal: a + client-side `"type": "function"` tool merely named `web_search`. + - `server_side_tools` — positive: a tools entry of `"type": "mcp"` (`server_url` / + `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, + `image_generation` used as a tool type, or an Anthropic `code_execution_*` / `text_editor_*` + / `computer_*` / `bash_*` tool type — any provider-executed tool other than web search. Not a + signal: `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, + `tool_choice` — client-side function tools never count. + - `projects` — positive: an `OpenAI-Project` header + (`default_headers={"OpenAI-Project": ...}`), a `project=` kwarg on the OpenAI client, or an + `OPENAI_PROJECT_ID`-style environment name read in code; in usage discovery, a + `project_filter` naming more than one project or a non-default project. Not a signal: + `OpenAI-Organization` alone; no header means the default project. + - `workspaces` — positive: an `anthropic-workspace-id` header or an `ANTHROPIC_WORKSPACE_ID`-style + environment name; in usage discovery, `workspace_ids` naming more than one workspace or a + non-default workspace. Not a signal: OpenRouter's own `workspace_id` scoping, which is an + OpenRouter account concept. +- **Derived by Design, never asked.** `mantle_only_model_or_api` is derived from the model id and + the required API against the access-path tables in `ai-openai-to-bedrock.md` / + `openai-on-bedrock.md`; it is not a scanner output and not a Clarify question. +- **Always asked, never detected.** `in_region_processing` is never code-detectable: nothing in a + call site says whether cross-Region inference is acceptable. Clarify always asks it together + with the Region. + ## Public comparison (read 2026-11-07) | Item | `bedrock-runtime` | `bedrock-mantle` | Source page | diff --git a/plugins/aws-startup-advisor/tests/test_bedrock_migration_assessment_core.py b/plugins/aws-startup-advisor/tests/test_bedrock_migration_assessment_core.py index b398f0ea1..a12ab8291 100644 --- a/plugins/aws-startup-advisor/tests/test_bedrock_migration_assessment_core.py +++ b/plugins/aws-startup-advisor/tests/test_bedrock_migration_assessment_core.py @@ -86,7 +86,7 @@ def test_every_neutral_fixture_is_accepted_without_a_sidecar(fixture_path: Path) result = BMA_VALIDATOR.validate_profile(_load(fixture_path)) assert result["ok"] is True assert set(result) == {"ok", "schema_version", "ai_sources", "gateways", "models_extractable", - "ai_framework_detected", "ai_vision_detected"} + "ai_framework_detected", "ai_vision_detected", "mantle_signals_detected"} @pytest.mark.parametrize("fixture_path", PROFILE_FIXTURES, ids=lambda p: p.parent.name) diff --git a/plugins/aws-startup-advisor/tests/test_mantle_selector_discovery.py b/plugins/aws-startup-advisor/tests/test_mantle_selector_discovery.py new file mode 100644 index 000000000..d5290a7ab --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_mantle_selector_discovery.py @@ -0,0 +1,185 @@ +"""Guard: Discover detects the code-visible Mantle-only selectors and records them as mantle_signals. + +Amazon Bedrock serves Responses `background=true` only on `bedrock-mantle`; `bedrock-runtime` +rejects it with a 400 (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, +read 2026-11-07). The same is true of server-side tools, Web Search, Projects and Workspaces, and +those five are the only selectors a code scan can see. These checks pin: the scanner steps in the +three app-code discovery files, the usage-API merges, the profile contracts (legacy per-workload +`workloads[].mantle_signals[]` and the provider-neutral top-level `mantle_signals[]`), the policy +file's "How a selector is learned" section, the validator's cross-check and summary, and the +bma-scan-openai-mantle-signals fixture whose negatives must never be cited. +""" +from __future__ import annotations + +import copy +import importlib.util +import json +import re +import sys +from pathlib import Path + +import pytest + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +SKILLS = PLUGIN_ROOT / "skills" +BMA = SKILLS / "bedrock-migration-assessment" + +SCANNERS = { + "gcp": SKILLS / "gcp-to-aws" / "references" / "phases" / "discover" / "discover-app-code.md", + "azure": SKILLS / "azure-to-aws" / "references" / "phases" / "discover" / "discover-app-code.md", + "bma": BMA / "references" / "phases" / "discover" / "discover-app-code.md", +} +USAGE_API_SIGNALS = [ + ("discover-openai-api.md", ["`projects`", "project_filter"]), + ("discover-anthropic-api.md", ["`workspaces`", "workspace_ids"]), + ("discover-openrouter-api.md", ["`:online`", "`web_search`", "NOT a `workspaces` signal"]), +] +POLICY_CANONICAL = SKILLS / "shared" / "ai" / "bedrock-endpoint-policy.md" +POLICY_VENDORED = [ + SKILLS / "gcp-to-aws" / "references" / "vendored" / "ai" / "bedrock-endpoint-policy.md", + SKILLS / "azure-to-aws" / "references" / "vendored" / "ai" / "bedrock-endpoint-policy.md", +] +CANONICAL_SCHEMA = PLUGIN_ROOT / "scripts" / "contracts" / "ai-workload-profile.schema.json" +BMA_SCHEMA = BMA / "references" / "contracts" / "ai-workload-profile.schema.json" +LEGACY_SCHEMA_DOCS = [ + SKILLS / "gcp-to-aws" / "references" / "shared" / "schema-discover-ai.md", + SKILLS / "azure-to-aws" / "references" / "shared" / "schema-discover-ai.md", +] +FIXTURE = PLUGIN_ROOT / "fixtures" / "bma-scan-openai-mantle-signals" +NEGATIVE_FIXTURE = PLUGIN_ROOT / "fixtures" / "bma-scan-openai-direct" + +DETECTABLE = ["background_inference", "web_search", "server_side_tools", "projects", "workspaces"] +NEVER_DETECTED = ["mantle_only_model_or_api", "in_region_processing"] +POSITIVE_CONSTRUCTS = ["web_search", "web_search_preview", '"type": "mcp"', "OpenAI-Project", "anthropic-workspace-id"] +NEGATIVE_CONSTRUCTS = ['"type": "function"', "store="] + +spec = importlib.util.spec_from_file_location("bma_mantle_validator", BMA / "scripts" / "validate-ai-profile.py") +VALIDATOR = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = VALIDATOR +spec.loader.exec_module(VALIDATOR) + + +def _load_fixture() -> dict: + return json.loads((FIXTURE / "ai-workload-profile.json").read_text(encoding="utf-8")) + + +# --------------------------------------------------------------------------- (a) scanners +@pytest.mark.parametrize("name", sorted(SCANNERS), ids=str) +def test_scanner_has_a_mantle_selector_step(name: str) -> None: + text = SCANNERS[name].read_text(encoding="utf-8") + assert re.search(r"^## Step [0-9.A-Z]+: Mantle-[Oo]nly [Ss]elector [Ss]ignals", text, re.M), f"{name}: no Mantle selector step heading" + for selector in DETECTABLE: + assert f"`{selector}`" in text, f"{name}: detectable selector {selector} missing" + assert "background=True" in text or "background: true" in text, f"{name}: background construct missing" + for construct in POSITIVE_CONSTRUCTS: + assert construct in text, f"{name}: positive construct {construct!r} missing" + for construct in NEGATIVE_CONSTRUCTS: + assert construct in text, f"{name}: negative construct {construct!r} missing" + assert "`in_region_processing` is never code-detectable" in text, f"{name}: must say in_region_processing is never detected" + assert "`mantle_only_model_or_api`" in text, f"{name}: must say mantle_only_model_or_api is not a scanner output" + assert "mantle_signals" in text, f"{name}: output key mantle_signals missing" + + +def test_legacy_scanners_write_per_workload_signals_and_bma_writes_top_level() -> None: + for name in ("gcp", "azure"): + assert "workloads[]" in SCANNERS[name].read_text(encoding="utf-8") + bma = SCANNERS["bma"].read_text(encoding="utf-8") + assert "a top-level `mantle_signals` array" in bma + assert "model_id" not in bma and "Terraform" not in bma and "gcloud" not in bma + + +# --------------------------------------------------------------------------- (b) usage-API merges +@pytest.mark.parametrize("skill", ["gcp-to-aws", "azure-to-aws"]) +@pytest.mark.parametrize(("filename", "needles"), USAGE_API_SIGNALS, ids=lambda v: v if isinstance(v, str) else "") +def test_usage_api_merges_name_their_signal(skill: str, filename: str, needles: list[str]) -> None: + path = SKILLS / skill / "references" / "phases" / "discover" / filename + text = path.read_text(encoding="utf-8") + step4 = re.sub(r"\s+", " ", text[text.index("## Step 4: Merge"):]) + assert "mantle_signals" in step4, f"{path}: Step 4 does not append to mantle_signals" + assert '"", "line": 0' in step4, f"{path}: usage-API signal must use the /line 0 sentinel" + for needle in needles: + assert needle in step4, f"{path}: {needle!r} missing from Step 4" + + +# --------------------------------------------------------------------------- (c) schema +def test_schema_copies_are_byte_identical_and_define_mantle_signal() -> None: + assert CANONICAL_SCHEMA.read_bytes() == BMA_SCHEMA.read_bytes(), "BMA schema copy drifted from the canonical" + schema = json.loads(CANONICAL_SCHEMA.read_text(encoding="utf-8")) + assert schema["properties"]["mantle_signals"]["items"] == {"$ref": "#/definitions/mantleSignal"} + assert "mantle_signals" not in schema["required"] + signal = schema["definitions"]["mantleSignal"] + assert signal["properties"]["selector"]["enum"] == DETECTABLE + assert signal["required"] == ["selector", "provider", "source_service", "model", "evidence"] + assert signal["additionalProperties"] is False + assert "mantle_signals" not in VALIDATOR.V11_FIELDS + for doc in LEGACY_SCHEMA_DOCS: + text = doc.read_text(encoding="utf-8") + assert "mantle_signals" in text, f"{doc}: legacy per-workload mantle_signals missing" + for selector in DETECTABLE: + assert selector in text, f"{doc}: selector {selector} missing" + + +# --------------------------------------------------------------------------- (d) policy +def test_policy_explains_how_each_selector_is_learned() -> None: + for path in [POLICY_CANONICAL, *POLICY_VENDORED]: + text = path.read_text(encoding="utf-8") + assert "## How a selector is learned" in text, f"{path}: section missing" + section = text[text.index("## How a selector is learned"):text.index("## Public comparison")] + for selector in DETECTABLE + NEVER_DETECTED: + assert f"`{selector}`" in section, f"{path}: {selector} not covered" + assert "inference-responses-api.html" in section + assert "never code-detectable" in section + assert "workloads[].mantle_signals[]" in section and "top-level `mantle_signals[]`" in section + + +# --------------------------------------------------------------------------- (e) fixture +def test_fixture_validates_and_cites_only_positive_lines() -> None: + profile = _load_fixture() + assert VALIDATOR.validate_profile(profile)["ok"] is True + source = (FIXTURE / "repo" / "app" / "llm.py").read_text(encoding="utf-8").splitlines() + joined = "\n".join(source) + for negative in ('"type": "function"', "store=True"): + assert negative in joined, f"fixture must contain the negative {negative!r}" + assert "Ignore previous instructions" not in json.dumps(profile) and "Ignore previous instructions" not in joined + selectors = [] + for signal in profile["mantle_signals"]: + selectors.append(signal["selector"]) + for entry in signal["evidence"]: + location, construct = entry["detail"].split(" ", 1) + path, line = location.split(":") + cited = (FIXTURE / "repo" / path).read_text(encoding="utf-8").splitlines()[int(line) - 1] + assert construct in cited, f"{entry['detail']}: cited line does not contain the construct" + for negative in ('"type": "function"', "store=True"): + assert negative not in cited, f"{entry['detail']}: cites a negative line" + assert sorted(selectors) == ["background_inference", "projects", "server_side_tools", "web_search"] + + +# --------------------------------------------------------------------------- (f) validator rejects +def test_validator_rejects_a_signal_on_an_unobserved_model() -> None: + profile = copy.deepcopy(_load_fixture()) + profile["mantle_signals"][0]["model"] = "gpt-4.1" + with pytest.raises(VALIDATOR.ProfileRejected) as excinfo: + VALIDATOR.validate_profile(profile) + assert excinfo.value.code == "unmatched_mantle_signal" + profile = copy.deepcopy(_load_fixture()) + profile["mantle_signals"][0]["source_service"] = "azure_openai" + with pytest.raises(VALIDATOR.ProfileRejected) as excinfo: + VALIDATOR.validate_profile(profile) + assert excinfo.value.code == "unmatched_mantle_signal" + profile = copy.deepcopy(_load_fixture()) + profile["mantle_signals"][0]["selector"] = "in_region_processing" + with pytest.raises(VALIDATOR.ProfileRejected) as excinfo: + VALIDATOR.validate_profile(profile) + assert excinfo.value.code == "schema_violation" + + +# --------------------------------------------------------------------------- (g) summary +def test_summary_lists_detected_signals() -> None: + detected = VALIDATOR.validate_profile(_load_fixture())["mantle_signals_detected"] + assert sorted(d["selector"] for d in detected) == ["background_inference", "projects", "server_side_tools", "web_search"] + for item in detected: + assert set(item) == {"selector", "provider", "source_service", "model", "detail"} + assert (item["provider"], item["source_service"], item["model"]) == ("openai", "openai_api", "gpt-4o") + assert item["detail"].startswith("app/llm.py:") + negative = json.loads((NEGATIVE_FIXTURE / "ai-workload-profile.json").read_text(encoding="utf-8")) + assert VALIDATOR.validate_profile(negative)["mantle_signals_detected"] == [] From aaf28442c2d68a14f4b8893c422b2f7255c38fc5 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 13:47:38 -0800 Subject: [PATCH 17/41] feat(aws-startup-advisor): ask Mantle-only requirements in Clarify (Q22b / Q10.5) and persist ai_constraints.mantle_requirements PR #585 defined the mantle_selector enum but nothing populated it from the user. Add one question block with four sub-questions (asynchronous/background inference; provider-run server-side or pre-configured tools incl. web search; customer-created Projects/Workspaces; strict single-Region processing + Region): - gcp-to-aws clarify-ai.md: Q22b (ESSENTIAL, Gate-2 AI batch, prefilled from workloads[].mantle_signals), Mantle-only signals line in the AI Context Summary, Preferences Output block, question budget 4 -> 5 globals - gcp-to-aws clarify-assemble.md: Q22b in the AI/agentic batch, its "use defaults" bullet and defaults-table row - gcp-to-aws / bedrock-migration-assessment clarify-ai-only.md: Q10.5 between Q10 and Q11 with identical table rows and Interpret line (BMA parity); BMA prefills from the validator summary's mantle_signals_detected; gcp fast path now presents Q10.5 (never dropped); Step 3 field row mantle_requirements - azure-to-aws clarify-ai.md: Q22b Category F bullet, Step 3 rows entry, Who-consumes row; azure clarify-ai-only.md: Q10.5 paragraph, fast path, Step 3 bullet, AI-only example block entry (shows web_search + region) - schema-preferences.md (gcp + BMA): mantle_requirements catalog rows (Q22b (AI-only: Q10.5) / ai_constraints Q10.5) - fixtures/bma-ai-only-clarify/preferences.json: mantle_requirements answer and Q10.5 in questions_asked - tests/test_mantle_requirements_clarify_contract.py: 13 new contract tests Wrapper: ai_constraints.mantle_requirements.value = [{ workload_id, selector, source, evidence?, region? }], selector in background_inference | server_side_tools | web_search | projects | workspaces | in_region_processing | none, source in detected | user | none; chosen_by user or default (source default: + report caveat), never extracted. Verification (plugins/aws-startup-advisor): - pytest tests/test_mantle_requirements_clarify_contract.py tests/test_bedrock_migration_assessment_core.py tests/test_validate_artifacts.py tests/test_azure_clarify_fast_path_contracts.py tests/test_estimate_clarify_volume_resolution.py tests/test_live_cli_ai_metrics.py: 243 passed - pytest tests (full plugin suite): 1475 passed, 222 subtests, 1 failed = tests/test_model_id_lint.py::test_real_plugin_passes flagging the untracked skills/llm-to-bedrock/scripts/.venv (pre-existing local env, not a tracked file) - validate-artifacts.py --fixtures: PASS 100 artifacts, 0 errors, baseline unchanged - validate-artifacts.py --run-dir fixtures/bma-ai-only-clarify --skill bedrock-migration-assessment --no-baseline: PASS - markdownlint-cli2: 0 errors on 823 files; sync-vendored --check PASS (64); check-skill-refs --check PASS (1525); tools/validate.py passed; git diff --check clean --- .../bma-ai-only-clarify/preferences.json | 9 +- .../phases/clarify/clarify-ai-only.md | 31 ++- .../references/phases/clarify/clarify-ai.md | 18 +- .../phases/clarify/clarify-ai-only.md | 30 ++- .../references/shared/schema-preferences.md | 5 +- .../phases/clarify/clarify-ai-only.md | 34 ++- .../references/phases/clarify/clarify-ai.md | 59 ++++- .../phases/clarify/clarify-assemble.md | 9 +- .../references/shared/schema-preferences.md | 1 + ...st_mantle_requirements_clarify_contract.py | 210 ++++++++++++++++++ 10 files changed, 390 insertions(+), 16 deletions(-) create mode 100644 plugins/aws-startup-advisor/tests/test_mantle_requirements_clarify_contract.py diff --git a/plugins/aws-startup-advisor/fixtures/bma-ai-only-clarify/preferences.json b/plugins/aws-startup-advisor/fixtures/bma-ai-only-clarify/preferences.json index 5b3729e32..88501b83d 100644 --- a/plugins/aws-startup-advisor/fixtures/bma-ai-only-clarify/preferences.json +++ b/plugins/aws-startup-advisor/fixtures/bma-ai-only-clarify/preferences.json @@ -4,7 +4,7 @@ "clarify_mode": "full", "timestamp": "2026-10-30T18:00:00Z", "discovery_artifacts": ["ai-workload-profile.json"], - "questions_asked": ["Q1", "Q1.5", "Q2", "Q4", "Q5", "Q6", "Q7", "Q8", "Q9", "Q10", "Q11"], + "questions_asked": ["Q1", "Q1.5", "Q2", "Q4", "Q5", "Q6", "Q7", "Q8", "Q9", "Q10", "Q10.5", "Q11"], "questions_defaulted": ["Q3"], "questions_extracted": [] }, @@ -94,6 +94,13 @@ "prompt": "Specialized features needed", "design_consequence": "Default recommendation from Q2 priority stands", "question_id": "Q10" + }, + "mantle_requirements": { + "value": [{ "workload_id": "*", "selector": "none", "source": "user" }], + "chosen_by": "user", + "prompt": "Mantle-only requirements (asynchronous inference, provider-run tools, Projects/Workspaces, single-Region processing)", + "design_consequence": "No Mantle-only requirement; bedrock-runtime selected for every workload with an explicit assumption in the design", + "question_id": "Q10.5" } }, "startup_constraints": { diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md index 2c3908516..fd4daa535 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md @@ -33,10 +33,12 @@ wording are Azure-swapped (Azure OpenAI, no Gemini). ## Step 1.5: Fast-path check If the AI usage looks simple (single model, non-agentic, no multi-provider/multi-model routing): -present ONLY Q1.5, Q2, Q3, Q4, Q11 (Q1 framework, Q5 model, Q6 capabilities are extracted; -Q7–Q10 default). **Q1.5 (compliance) and Q11 (Activate) are never dropped** — always PRESENTED, -never silently omitted. "Use defaults for the rest" applies documented defaults (compliance → -`["unknown"]` + a `metadata.report_caveats[]` entry, never a silent "none"; Activate → `unknown`). +present ONLY Q1.5, Q2, Q3, Q4, Q10.5, Q11 (Q1 framework, Q5 model, Q6 capabilities are extracted; +Q7–Q10 default). **Q1.5 (compliance), Q10.5 (Mantle-only requirements) and Q11 (Activate) are +never dropped** — always PRESENTED, never silently omitted. "Use defaults for the rest" applies +documented defaults (compliance → `["unknown"]` + a `metadata.report_caveats[]` entry, never a +silent "none"; Mantle-only requirements → the `none` entry + report caveat, `bedrock-runtime` +assumed; Activate → `unknown`). Then skip to Step 3. Otherwise continue to Step 1.75 → Step 2. ## Step 1.75: Mini assumption sheet @@ -167,6 +169,16 @@ moderate / complex). → `ai_complexity`. Default `"moderate"`. **Q10 — Specia (function calling / ultra-long-context / extended-thinking / prompt-caching / RAG / agentic / real-time / image-gen / speech). → `ai_critical_feature`. Default none. +**Q10.5 — Mantle-only requirements** (a: asynchronous/background inference; b: provider-run +server-side or pre-configured tools incl. web search; c: customer-created Projects/Workspaces; +d: strict single-Region processing + Region). Prefill from `workloads[].mantle_signals` and +confirm; otherwise ask. Never dropped. → `ai_constraints.mantle_requirements` (array of +`{ workload_id, selector, source, evidence?, region? }`; `selector` ∈ `background_inference | +server_side_tools | web_search | projects | workspaces | in_region_processing | none`, `source` ∈ +`detected | user | none`, `region` required for `in_region_processing`). Default: one +`{ "workload_id": "*", "selector": "none", "source": "none" }` entry + report caveat +("Mantle-only requirements were not confirmed by the user; bedrock-runtime assumed"). + **Q11 — AWS Activate credits** (**never dropped**; ≡ full-flow Q27). Never infer funding stage from Q3 spend. Options: have credits / self-funded (→ Activate Founders up to $5K) / VC-backed (→ Activate Portfolio up to $200K, needs Org ID) / don't know. Escalations: `>$10K`→ @@ -188,8 +200,8 @@ Write `$MIGRATION_DIR/preferences.json`: - `startup_constraints.startup_program_status` (Q11). - `ai_constraints`: `ai_framework`, `ai_priority`, `ai_monthly_spend`, `cross_cloud` (Q4 — unique to AI-only), `ai_model_baseline`, `ai_vision`, `ai_token_volume`, `ai_latency`, - `ai_complexity`, `ai_critical_feature`, `ai_capabilities_required` (derived from - `capabilities_summary`). + `ai_complexity`, `ai_critical_feature`, `mantle_requirements` (Q10.5), + `ai_capabilities_required` (derived from `capabilities_summary`). - Persist the confirmed `workloads[]` (top-level) exactly as `clarify-ai.md` specifies — each entry carries `workload_id`, `model_id`, `sdk_method`, `capability`, `capability_confidence`, `structured_output`, `call_sites`, `target_bedrock_model`, `priority`, `latency_tier`. This is @@ -273,6 +285,13 @@ discovery is never indistinguishable from a promoted default. No nulls. Delete }, "ai_capabilities_required": { // REQUIRED "value": [], "chosen_by": "derived", "prompt": "", "design_consequence": "", "question_id": "Q25" + }, + "mantle_requirements": { // optional on older runs; Design treats absence as no requirement recorded + "value": [ + { "workload_id": "wl-1", "selector": "web_search", "source": "detected", "evidence": "app.py:14 tools type web_search" }, + { "workload_id": "*", "selector": "in_region_processing", "source": "user", "region": "us-east-1" } + ], + "chosen_by": "user", "prompt": "", "design_consequence": "", "question_id": "Q10.5" } }, "workloads": [ // REQUIRED — [] when none; design-ai reads this array, not the profile diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md index 38c6a5275..ec39f6cac 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md @@ -66,6 +66,20 @@ models detected, capabilities, integration pattern + SDK, gateway/router, framew text-only. → `ai_vision`. - **Q21 — Response speed.** PROPOSED, default `"important"`. → `ai_latency`. - **Q22 — Task complexity.** PROPOSED, default `"moderate"`. → `ai_complexity`. +- **Q22b — Mantle-only requirements.** DETECTED from `workloads[].mantle_signals` when present + (row `source: "workloads[].mantle_signals"`, user confirms on the sheet), otherwise + **ESSENTIAL** (asked in the Gate-2 batch; no default on purpose — the answer decides the + endpoint). Four sub-questions (a: asynchronous/background inference; b: provider-run + server-side or pre-configured tools incl. web search, hosted MCP servers, code execution, + file search — client-side function tools do not count; c: customer-created Projects (OpenAI) + / Workspaces (Anthropic); d: strict single-Region processing + which Region). → + `ai_constraints.mantle_requirements` (array of `{ workload_id, selector, source, evidence?, + region? }`; selector/source vocab as gcp Q22b: `selector` ∈ `background_inference | + server_side_tools | web_search | projects | workspaces | in_region_processing | none`, `source` + ∈ `detected | user | none`; `region` required for `in_region_processing`). "Use defaults for + the rest" writes `[{ "workload_id": "*", "selector": "none", "source": "none" }]` with + `chosen_by: "default"`, `source: "default:Q22b"` and the report caveat "Mantle-only + requirements were not confirmed by the user; bedrock-runtime assumed". ### Category G — Agentic (fires ONLY when `agentic_profile.is_agentic == true`) @@ -173,7 +187,8 @@ and is REQUIRED on `DETECTED` applies here the same as the infra flow. "value": false, "default": false } - } + }, + "mantle_requirements": { "disposition": "ESSENTIAL", "value": [], "default": [] } // DETECTED with source "workloads[].mantle_signals" when Discover found signals; value = [{ workload_id, selector, source, evidence?, region? }] }, "workloads": [] // NOT a row — the confirmed array itself (persist rule above), same as clarify-ai-only.md's workloads[] } @@ -204,6 +219,7 @@ and is REQUIRED on `DETECTED` applies here the same as the infra flow. | -------------------------------------------------------------------------------------- | --------------------------------------------------- | | `workloads[]`, `ai_priority`, `ai_critical_feature`, `ai_latency`, `ai_model_baseline` | `design-ai.md` model selection + override hierarchy | | `ai_constraints.agentic.migration_approach` | `design-ai.md` Step 0.6 agentic routing | +| `ai_constraints.mantle_requirements` | `design-ai.md` Step 0.8 endpoint selection | | `ai_token_volume`, `ai_monthly_spend` | `estimate-ai.md` token tiers + ROI | | `startup_program_status`, `ai_monthly_spend` | `generate-artifacts-ai.md` STARTUP_PROGRAMS.md | diff --git a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md index 7dd9876fb..b42a6a3ec 100644 --- a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md @@ -264,7 +264,7 @@ After the user responds to Batch 1: ``` Got it — your AI strategy preferences are saved. -Last section — 6 questions about your technical requirements, then your answers are saved. +Last section — 7 questions about your technical requirements, then your answers are saved. You can answer each, skip individual ones, or say "use defaults for the rest." --- Technical Requirements --- @@ -350,6 +350,33 @@ Interpret → `ai_complexity`. Default: 2 → `"moderate"`. Interpret → `ai_critical_feature`. Default: 10 → no override. +## Q10.5 — Mantle-only requirements (asynchronous inference, provider-run tools, Projects/Workspaces, single-Region processing) + +Amazon Bedrock serves most applications on `bedrock-runtime`; a few needs exist only on `bedrock-mantle`. Answer per workload, or "none" / "all". + +_Never skipped:_ ask it in Batch 2 unless the user says "use defaults for the rest". Prefill from the validator summary's `mantle_signals_detected` (selector, model, detail); confirm each with the user: show each detected signal as `[model] — [selector] — [detail]` and ask the user to confirm or correct before writing; a declined signal is dropped. When the summary reports no signals, ask the four sub-questions outright. + +> a. Does any workload rely on asynchronous/background inference — long-running requests you submit now and poll for later? +> +> b. Does any workload use server-side or pre-configured tools run by the provider — including web search, hosted MCP servers, code execution or file search? (Your own client-side function tools do not count.) +> +> c. Does any workload use customer-created Projects (OpenAI) or Workspaces (Anthropic) for isolation or cost tracking? +> +> d. Does any workload have a strict requirement that inference stays in one AWS Region? If so, which Region? + +| Answer | Recommendation Impact | +| --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| a = yes | `background_inference` → Mantle for that workload (`bedrock-runtime` rejects `background=true` with 400 — [Responses API on Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html), read 2026-11-07) | +| b = yes, web search | `web_search` → Mantle (Runtime has no Web Search tool) | +| b = yes, other provider-run tools | `server_side_tools` → Mantle | +| c = yes | `projects` (OpenAI source) / `workspaces` (Anthropic source) → Mantle (Runtime exposes only the default project) | +| d = yes + Region | `in_region_processing` with `region` — applied by Design only when `bedrock-runtime` serves that model in that Region through a CRIS profile alone; otherwise `bedrock-runtime` in-Region | +| none | No selector; Design records `bedrock-runtime` and states the assumption | + +Interpret → `ai_constraints.mantle_requirements`: array of `{ workload_id, selector, source, evidence?, region? }` — `selector` ∈ `background_inference | server_side_tools | web_search | projects | workspaces | in_region_processing | none`, `source` ∈ `detected | user | none`. Default: `[{ "workload_id": "*", "selector": "none", "source": "none" }]` with `chosen_by: "default"` and the report caveat "Mantle-only requirements were not confirmed by the user; bedrock-runtime assumed". + +`workload_id` is the `workloads[].workload_id` the answer applies to, or `*` for every workload (always `*` when the profile has no `workloads[]`; the summary's signals are per model, so use `*` unless the user names a workload). `source: "detected"` = prefilled from a summary signal and confirmed by the user (copy the signal's `detail` into `evidence`); `source: "user"` = answered without a signal (an explicit "none" answer records `{ "workload_id": "*", "selector": "none", "source": "user" }`); `source: "none"` = defaulted. `region` is required when `selector` is `in_region_processing`. The constraint wrapper uses `chosen_by: "user"` when answered or confirmed, `chosen_by: "default"` + `source: "default:Q10.5"` when skipped (append the caveat to `metadata.report_caveats[]`) — never `extracted`. `mantle_only_model_or_api` is never written here; Design derives it from the model catalog. + ## Q11 — Have you applied for AWS Activate credits? **Activate eligibility:** Pre-Series B, founded in the last 10 years, AWS Account on Paid Tier Plan, and either new to Activate Credits or requesting more credits than previously received. AI-only migrations are exactly the workloads Activate credits offset — Bedrock usage (Claude, Llama, Nova) is credit-eligible. **Never infer funding stage or Activate tier from Q3 spend.** @@ -406,6 +433,7 @@ Write `$MIGRATION_DIR/preferences.json`: | `ai_latency` | `ai_constraints.ai_latency` | From Q8 | | `ai_complexity` | `ai_constraints.ai_complexity` | From Q9 | | `ai_critical_feature` | `ai_constraints.ai_critical_feature` | From Q10 | +| `mantle_requirements` | `ai_constraints.mantle_requirements` | From Q10.5 — array of `{ workload_id, selector, source, evidence?, region? }`; Design derives `mantle_selector` from it | `ai_capabilities_required` is omitted: Q6 and Q10 already record the requirement. diff --git a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/shared/schema-preferences.md b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/shared/schema-preferences.md index f65c8e2d2..c92c45a20 100644 --- a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/shared/schema-preferences.md +++ b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/shared/schema-preferences.md @@ -14,7 +14,7 @@ Every key in `design_constraints`, `ai_constraints`, and `startup_constraints` ( | `chosen_by` | string | yes | `"user"` \| `"default"` \| `"extracted"` \| `"derived"` | | `prompt` | string | yes | Question shown to the user, or a short detection label when skipped | | `design_consequence` | string | yes | One sentence: how this choice shaped the AWS design, estimate, or plan | -| `question_id` | string | no | AI-only catalog ID (`Q1`–`Q11`, `Q1.5`) when mapped; omit for derived values | +| `question_id` | string | no | AI-only catalog ID (`Q1`–`Q11`, `Q1.5`, `Q10.5`) when mapped; omit for derived values | | `source` | string | no | Raw provenance signal. Required when `chosen_by` is `"extracted"` (prefix `ai-profile:`) or `"default"` (`"default:"`). Omit for `"user"` and `"derived"`. | **Do not write null values.** Omit keys that produce no constraint. @@ -81,7 +81,7 @@ Use the **Impact** row for the selected answer from `clarify-ai-only.md` when th } ``` -`metadata.report_caveats` is present only when a caveat was added (Q1.5 default). +`metadata.report_caveats` is present only when a caveat was added (Q1.5 or Q10.5 default). --- @@ -101,6 +101,7 @@ Use the **Impact** row for the selected answer from `clarify-ai-only.md` when th | `ai_latency` | `ai_constraints` | Q8 | Response speed importance | `[value]` adds latency success criteria and model filtering | | `ai_complexity` | `ai_constraints` | Q9 | AI task complexity | `[value]` affects recommended model size and agentic path | | `ai_critical_feature` | `ai_constraints` | Q10 | Specialized features needed | `[value]` gates the model shortlist and capability validation | +| `mantle_requirements` | `ai_constraints` | Q10.5 | Mantle-only requirements (asynchronous inference, provider-run tools, Projects/Workspaces, single-Region processing) | `[value]` entries with a selector route that workload to `bedrock-mantle`; no entry keeps `bedrock-runtime` with an explicit assumption | | `startup_program_status` | `startup_constraints` | Q11 | Have you applied for AWS Activate credits? | `[value]` triggers the Activate credits callout in the plan | --- diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md index bbf691b8c..0553e6446 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md @@ -51,9 +51,9 @@ Check `$MIGRATION_DIR/` for existing state: If `migration-preview.json` exists and `ai_complexity_signal == "likely_simple"` (single model, non-agentic, no multi-provider, no multi-model routing): -> "Your AI migration looks straightforward — one model swapping to Bedrock. I only need 5 quick answers to complete your migration plan." +> "Your AI migration looks straightforward — one model swapping to Bedrock. I only need 6 quick answers to complete your migration plan." -Present **only Q1.5, Q2, Q3, Q4, Q11** (Q1 framework is extracted; Q5 model is extracted; Q6 capabilities are extracted; Q7–Q10 use defaults). **Q1.5 (compliance) and Q11 (Activate status) are never dropped from the fast path** — "never dropped" means always PRESENTED: the fast-path question set must include them; they are never silently omitted or auto-answered. An explicit user "use defaults for the rest" still applies their documented defaults (compliance → `["unknown"]` + report caveat — never a silent "none", matching full-flow Q2 semantics; Activate → `unknown` + neutral copy) — that is the sanctioned default path, same as full-flow Q27. After answering, skip directly to Step 3. +Present **only Q1.5, Q2, Q3, Q4, Q10.5, Q11** (Q1 framework is extracted; Q5 model is extracted; Q6 capabilities are extracted; Q7–Q10 use defaults). **Q1.5 (compliance), Q10.5 (Mantle-only requirements) and Q11 (Activate status) are never dropped from the fast path** — "never dropped" means always PRESENTED: the fast-path question set must include them; they are never silently omitted or auto-answered. An explicit user "use defaults for the rest" still applies their documented defaults (compliance → `["unknown"]` + report caveat — never a silent "none", matching full-flow Q2 semantics; Mantle-only requirements → the `none` entry + report caveat, `bedrock-runtime` assumed; Activate → `unknown` + neutral copy) — that is the sanctioned default path, same as full-flow Q27. After answering, skip directly to Step 3. If `ai_complexity_signal` is `"standard"` or `"complex"`, or `migration-preview.json` is absent, continue to Step 1.75 (mini assumption sheet), then Step 2. @@ -243,7 +243,7 @@ After the user responds to Batch 1: ``` Got it — your AI strategy preferences are saved. -Last section — 6 questions about your technical requirements, then we're ready to design. +Last section — 7 questions about your technical requirements, then we're ready to design. You can answer each, skip individual ones, or say "use defaults for the rest." --- Technical Requirements --- @@ -318,6 +318,33 @@ Same decision logic as Q17 in `clarify-ai.md`. Interpret → `ai_critical_feature`. Default: 10 → no override. +## Q10.5 — Mantle-only requirements (asynchronous inference, provider-run tools, Projects/Workspaces, single-Region processing) + +Amazon Bedrock serves most applications on `bedrock-runtime`; a few needs exist only on `bedrock-mantle`. Answer per workload, or "none" / "all". + +_Never skipped:_ ask it in Batch 2 unless the user says "use defaults for the rest". Prefill from `workloads[].mantle_signals` when present: show each detected signal as `[workload] — [selector] — [file:line construct]` and ask the user to confirm or correct before writing; a declined signal is dropped. When the profile has no `workloads[]`, no signals, or no application code was scanned, ask the four sub-questions outright. + +> a. Does any workload rely on asynchronous/background inference — long-running requests you submit now and poll for later? +> +> b. Does any workload use server-side or pre-configured tools run by the provider — including web search, hosted MCP servers, code execution or file search? (Your own client-side function tools do not count.) +> +> c. Does any workload use customer-created Projects (OpenAI) or Workspaces (Anthropic) for isolation or cost tracking? +> +> d. Does any workload have a strict requirement that inference stays in one AWS Region? If so, which Region? + +| Answer | Recommendation Impact | +| --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| a = yes | `background_inference` → Mantle for that workload (`bedrock-runtime` rejects `background=true` with 400 — [Responses API on Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html), read 2026-11-07) | +| b = yes, web search | `web_search` → Mantle (Runtime has no Web Search tool) | +| b = yes, other provider-run tools | `server_side_tools` → Mantle | +| c = yes | `projects` (OpenAI source) / `workspaces` (Anthropic source) → Mantle (Runtime exposes only the default project) | +| d = yes + Region | `in_region_processing` with `region` — applied by Design only when `bedrock-runtime` serves that model in that Region through a CRIS profile alone; otherwise `bedrock-runtime` in-Region | +| none | No selector; Design records `bedrock-runtime` and states the assumption | + +Interpret → `ai_constraints.mantle_requirements`: array of `{ workload_id, selector, source, evidence?, region? }` — `selector` ∈ `background_inference | server_side_tools | web_search | projects | workspaces | in_region_processing | none`, `source` ∈ `detected | user | none`. Default: `[{ "workload_id": "*", "selector": "none", "source": "none" }]` with `chosen_by: "default"` and the report caveat "Mantle-only requirements were not confirmed by the user; bedrock-runtime assumed". + +`workload_id` is the `workloads[].workload_id` the answer applies to, or `*` for every workload (always `*` when the profile has no `workloads[]`). `source: "detected"` = prefilled from a Discover signal and confirmed by the user (copy the signal's `file:line construct` into `evidence`); `source: "user"` = answered without a code signal (an explicit "none" answer records `{ "workload_id": "*", "selector": "none", "source": "user" }`); `source: "none"` = defaulted. `region` is required when `selector` is `in_region_processing`. The constraint wrapper uses `chosen_by: "user"` when answered or confirmed, `chosen_by: "default"` + `source: "default:Q10.5"` when skipped (append the caveat to `metadata.report_caveats[]`) — never `extracted`. `mantle_only_model_or_api` is never written here; Design derives it from the model catalog. + ## Q11 — Have you applied for AWS Activate credits? > **Numbering note:** AI-only Q11 ≡ full-flow **Q27** (startup programs). It is unrelated to the full flow's Q11 (Cloud Run spend) or Q11b (Graviton) — the two flows number independently. @@ -376,6 +403,7 @@ Write `$MIGRATION_DIR/preferences.json`: | `ai_latency` | `ai_constraints.ai_latency` | From Q8 | | `ai_complexity` | `ai_constraints.ai_complexity` | From Q9 | | `ai_critical_feature` | `ai_constraints.ai_critical_feature` | From Q10 | +| `mantle_requirements` | `ai_constraints.mantle_requirements` | From Q10.5 — array of `{ workload_id, selector, source, evidence?, region? }`; Design derives `mantle_selector` from it | | `ai_capabilities_required` | `ai_constraints.ai_capabilities_required` | Derived from `capabilities_summary` | Each constraint carries the FULL clarify.md field shape — `value`, `chosen_by` (`user`|`extracted`|`default`|`derived`), `prompt`, `design_consequence`, and `source`/`question_id` per the clarify.md source-field rules. (The short `{ value, chosen_by }` form shown above is an abbreviation, not the schema.) No nulls. All schema rules from `clarify.md` apply, with two AI-only bindings: `metadata.clarify_mode` is `"fast_path"` when Step 1.5 fired, `"full"` for the two-batch flow; and this flow's `questions_extracted` is the full flow's `questions_skipped_extracted` (downstream consumers accept both names). diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai.md index 78c385a62..2b0b6b051 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai.md @@ -41,6 +41,7 @@ Before presenting questions, show: > **Integration pattern:** [from `integration.pattern`; if `unknown`, say **Unknown (IaC-only)**] via [from `integration.primary_sdk`, or **not determined**] > **Gateway/router:** [from `integration.gateway_type`, or "None (direct SDK)"] > **Frameworks:** [from `integration.frameworks`, or "None"] +> **Mantle-only signals:** [from `workloads[].mantle_signals`: one line per signal as `[workload] — [selector] — [file:line construct]`, or "None detected" — confirmed in Q22b] --- @@ -115,7 +116,7 @@ Before presenting Q16–Q22, show the detected workloads and proposed Bedrock ta **Zero-workload case:** If no AI workloads detected and user doesn't report any, write `"workloads": []` to preferences.json. Design emits empty `design_blocks[]` for this case. -5. **Question budget:** 4 global questions (Q14, Q15, framework, spend) + at most 2 per medium/low workload. For an app with 3 high-confidence workloads: 4 questions total, 0 per-workload. For an app with 2 high + 1 medium: 4 + 2 = 6 questions max. +5. **Question budget:** 5 global questions (Q14, Q15, Q22b, framework, spend) + at most 2 per medium/low workload. For an app with 3 high-confidence workloads: 5 questions total, 0 per-workload. For an app with 2 high + 1 medium: 5 + 2 = 7 questions max. **Single-workload fallback:** If `workloads[]` has exactly 1 entry or is empty, skip the confirmation table and proceed with the existing Q16–Q22 flow below. @@ -437,6 +438,37 @@ Interpret → `ai_complexity`. Default: 2 → `"moderate"`. --- +## Q22b — Mantle-only requirements (asynchronous inference, provider-run tools, Projects/Workspaces, single-Region processing) + +**Disposition: ESSENTIAL** — asked in the Gate-2 AI batch (`clarify-assemble.md`), never placed on the assumption sheet: the answer decides the endpoint, so there is no safe default to propose. Prefill from `workloads[].mantle_signals` when present: show each detected signal as `[workload] — [selector] — [file:line construct]` and ask the user to confirm or correct before writing; a declined signal is dropped. When `metadata.profile_source` is `iac_vertex`, the profile is billing-only, or the user declined the code scan, say so in the lead-in — "No application code was scanned, so I cannot infer this" — and ask the four sub-questions outright. + +**Multi-workload rule:** ask once ("does any workload…"), then list the affected workload numbers from the confirmation table → one entry per `workload_id`. + +Amazon Bedrock serves most applications on `bedrock-runtime`; a few needs exist only on `bedrock-mantle`. Answer per workload, or "none" / "all". + +> a. Does any workload rely on asynchronous/background inference — long-running requests you submit now and poll for later? +> +> b. Does any workload use server-side or pre-configured tools run by the provider — including web search, hosted MCP servers, code execution or file search? (Your own client-side function tools do not count.) +> +> c. Does any workload use customer-created Projects (OpenAI) or Workspaces (Anthropic) for isolation or cost tracking? +> +> d. Does any workload have a strict requirement that inference stays in one AWS Region? If so, which Region? + +| Answer | Recommendation Impact | +| --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| a = yes | `background_inference` → Mantle for that workload (`bedrock-runtime` rejects `background=true` with 400 — [Responses API on Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html), read 2026-11-07) | +| b = yes, web search | `web_search` → Mantle (Runtime has no Web Search tool) | +| b = yes, other provider-run tools | `server_side_tools` → Mantle | +| c = yes | `projects` (OpenAI source) / `workspaces` (Anthropic source) → Mantle (Runtime exposes only the default project) | +| d = yes + Region | `in_region_processing` with `region` — applied by Design only when `bedrock-runtime` serves that model in that Region through a CRIS profile alone; otherwise `bedrock-runtime` in-Region | +| none | No selector; Design records `bedrock-runtime` and states the assumption | + +Interpret → `ai_constraints.mantle_requirements`: array of `{ workload_id, selector, source, evidence?, region? }` — `selector` ∈ `background_inference | server_side_tools | web_search | projects | workspaces | in_region_processing | none`, `source` ∈ `detected | user | none`. Default: `[{ "workload_id": "*", "selector": "none", "source": "none" }]` with `chosen_by: "default"` and the report caveat "Mantle-only requirements were not confirmed by the user; bedrock-runtime assumed". + +`workload_id` is the `workloads[].workload_id` the answer applies to, or `*` for every workload (always `*` when the profile has no `workloads[]`). `source: "detected"` = prefilled from a Discover signal and confirmed by the user (copy the signal's `file:line construct` into `evidence`); `source: "user"` = answered without a code signal (an explicit "none" answer records `{ "workload_id": "*", "selector": "none", "source": "user" }`); `source: "none"` = defaulted. `region` is required when `selector` is `in_region_processing`. The constraint wrapper uses `chosen_by: "user"` when answered or confirmed, `chosen_by: "default"` + `source: "default:Q22b"` when skipped (append the caveat to `metadata.report_caveats[]`) — never `extracted`. `mantle_only_model_or_api` is never written here; Design derives it from the model catalog. + +--- + ## Category G — Agentic Workflows (If `agentic_profile` exists in `ai-workload-profile.json`) _Fire when:_ `ai-workload-profile.json` contains `agentic_profile` with `is_agentic: true`. @@ -617,6 +649,31 @@ Category G answers are stored in `preferences.json` → `ai_constraints.agentic` - `task_duration` — Determines AgentCore Runtime recommendation and session limit warnings - `incremental_migration` — Determines whether incremental migration artifacts are generated +## Preferences Output — `ai_constraints.mantle_requirements` + +Q22b answers are stored in `preferences.json` → `ai_constraints.mantle_requirements` as a standard constraint wrapper whose `value` is the entry array: + +```json +{ + "ai_constraints": { + "mantle_requirements": { + "value": [ + { "workload_id": "wl_9532e7", "selector": "projects", "source": "detected", "evidence": "src/ai/openai_client.py:9 default_headers OpenAI-Project" }, + { "workload_id": "*", "selector": "in_region_processing", "source": "user", "region": "us-east-1" } + ], + "chosen_by": "user", + "prompt": "Mantle-only requirements (asynchronous inference, provider-run tools, Projects/Workspaces, single-Region processing)", + "design_consequence": "Workloads with a selector route to bedrock-mantle; every other workload stays on bedrock-runtime", + "question_id": "Q22b" + } + } +} +``` + +**Field contract (consumed by Design phase):** + +- `value[]` — one `{ workload_id, selector, source, evidence?, region? }` entry per confirmed requirement (vocabulary in Q22b's Interpret line); consumed by `design-ai.md` Step 0.8 to derive `bedrock_models[].mantle_selector` for the matching workload(s). The default entry (`selector: "none"`, `source: "none"`, `chosen_by: "default"`, `source: "default:Q22b"` on the wrapper) makes Design record `bedrock-runtime` with the assumption sentence and the report caveat. + ## Status — build step 5 (restructure) Implemented. Frontmatter and fragment framing added for consistency with the other diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-assemble.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-assemble.md index 24160745a..df5d30fa4 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-assemble.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-assemble.md @@ -225,7 +225,7 @@ Present the essential questions (from each fragment's disposition, plus any rows text, context, and options from the fragment files. Typical total: 2–7. **Batch composition:** core first (Q1/Q2/Q3/Q3.5/Q7 + conflicts, from `clarify-global.md`), -then AI/agentic (Q15, Q23–Q25, multi-workload confirmations, from `clarify-ai.md`). **Q27 is +then AI/agentic (Q15, Q22b, Q23–Q25, multi-workload confirmations, from `clarify-ai.md`). **Q27 is not batched** — ask it as its own short follow-up after the last batch, per the presentation rule in `clarify-ai.md` Q27 (it's a funding question, not a technical one). Write `preferences-draft.json` between batches (same schema as `preferences.json` plus @@ -271,6 +271,12 @@ default genuinely does not exist: skips it, record `startup_program_status: "unknown"` with `chosen_by: "default"` and `source: "default:Q27"`; downstream artifacts must use neutral Activate copy (both tiers, no "your status: eligible_*"). +- Q22b (Mantle-only requirements) defaults to the single entry + `{ "workload_id": "*", "selector": "none", "source": "none" }` in + `ai_constraints.mantle_requirements` with `chosen_by: "default"`, `source: "default:Q22b"`, + and appends "Mantle-only requirements were not confirmed by the user; bedrock-runtime + assumed" to `metadata.report_caveats[]` (Design then records `bedrock-runtime` with the + assumption sentence). - Q3.5, Q23–Q25, and unresolved multi-instance conflicts fall back to their documented defaults (Q3.5 → 5; Q23 → framework-based; Q24 → session; Q25 → medium; conflicts → most conservative posture) with `chosen_by: "default"`. @@ -693,6 +699,7 @@ skips, "use defaults for the rest", and the fast paths. | Q20 — Input types | 1 (text only) | no constraint | | Q21 — AI latency | 2 (important) | `ai_latency: "important"` | | Q22 — Task complexity | 2 (moderate) | `ai_complexity: "moderate"` | +| Q22b — Mantle-only requirements | none (bedrock-runtime assumed, report caveat) | `ai_constraints.mantle_requirements` default entry | | Q23 — Agentic approach | _(framework-based auto-detect; see `clarify-ai.md`)_ | `ai_constraints.agentic.migration_approach` | | Q24 — Agent memory | 2 (session) | `ai_constraints.agentic.memory_requirement: "session"` | | Q25 — Task duration | 2 (medium) | `ai_constraints.agentic.task_duration: "medium"` | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-preferences.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-preferences.md index 929d14beb..28bc20fc8 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-preferences.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-preferences.md @@ -90,6 +90,7 @@ Use when assembling Step 5. Replace `[value]` with the interpreted constraint. | `ai_vision` | Q20 | What input types does your AI use? | `[value]` requires vision-capable Bedrock models when not text-only | | `ai_latency` | Q21 | How important is AI response latency? | `[value]` adds P95 latency success criteria and model filtering | | `ai_complexity` | Q22 | How complex are your AI tasks? | `[value]` affects recommended model size and agentic path | +| `mantle_requirements` | Q22b (AI-only: Q10.5) | Mantle-only requirements (asynchronous inference, provider-run tools, Projects/Workspaces, single-Region processing) | `[value]` entries with a selector route that workload to `bedrock-mantle`; no entry keeps `bedrock-runtime` with an explicit assumption | | `startup_program_status` | Q27 (AI-only: Q11) | Are you eligible for AWS startup programs? | `[value]` triggers Activate credits callout in report and docs | | `ai_capabilities_required` | — | Derived from detected capabilities and your answers | Union of required capabilities (`[value]`) enforced in Bedrock model mapping | diff --git a/plugins/aws-startup-advisor/tests/test_mantle_requirements_clarify_contract.py b/plugins/aws-startup-advisor/tests/test_mantle_requirements_clarify_contract.py new file mode 100644 index 000000000..e7de7cd32 --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_mantle_requirements_clarify_contract.py @@ -0,0 +1,210 @@ +"""Guard: Clarify asks the Mantle-only requirements and persists them. + +PR #585 made `bedrock-runtime` the default endpoint and defined the seven-value +`mantle_selector` enum that every `bedrock-mantle` design row must record, but +nothing populated it: Discover can only see five selectors in code, Terraform or +billing-only runs scan no code at all, and `in_region_processing` is never +code-detectable. Design therefore had no user-confirmed input to choose an +endpoint from. The fix adds one question block — full-flow `Q22b` (gcp-to-aws and +azure-to-aws `clarify-ai.md`) and AI-only `Q10.5` (gcp-to-aws, azure-to-aws and +bedrock-migration-assessment `clarify-ai-only.md`) — that prefills Discover's +`mantle_signals` for confirmation or asks outright, and writes the answer as the +wrapper constraint `ai_constraints.mantle_requirements`. + +Public source for the one hard capability gap the question names: +`bedrock-runtime` rejects `background=true` with a 400 +(https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, +read 2026-11-07). +""" +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +SKILLS = PLUGIN_ROOT / "skills" + +GCP_CLARIFY_AI = SKILLS / "gcp-to-aws" / "references" / "phases" / "clarify" / "clarify-ai.md" +GCP_CLARIFY_AI_ONLY = SKILLS / "gcp-to-aws" / "references" / "phases" / "clarify" / "clarify-ai-only.md" +GCP_CLARIFY_ASSEMBLE = SKILLS / "gcp-to-aws" / "references" / "phases" / "clarify" / "clarify-assemble.md" +AZURE_CLARIFY_AI = SKILLS / "azure-to-aws" / "references" / "phases" / "clarify" / "clarify-ai.md" +AZURE_CLARIFY_AI_ONLY = SKILLS / "azure-to-aws" / "references" / "phases" / "clarify" / "clarify-ai-only.md" +BMA_CLARIFY_AI_ONLY = SKILLS / "bedrock-migration-assessment" / "references" / "phases" / "clarify" / "clarify-ai-only.md" +GCP_PREFERENCES_SCHEMA = SKILLS / "gcp-to-aws" / "references" / "shared" / "schema-preferences.md" +BMA_PREFERENCES_SCHEMA = SKILLS / "bedrock-migration-assessment" / "references" / "shared" / "schema-preferences.md" +BMA_RUN_FIXTURE = PLUGIN_ROOT / "fixtures" / "bma-ai-only-clarify" / "preferences.json" + +CLARIFY_FILES = [GCP_CLARIFY_AI, GCP_CLARIFY_AI_ONLY, AZURE_CLARIFY_AI, AZURE_CLARIFY_AI_ONLY, BMA_CLARIFY_AI_ONLY] + +CONSTRAINT = "ai_constraints.mantle_requirements" +ENTRY_KEYS = {"workload_id", "selector", "source", "evidence", "region"} +# The seven policy values (tests/test_endpoint_policy_contract.py) minus the one Design derives, plus the explicit "none". +CLARIFY_SELECTORS = {"background_inference", "server_side_tools", "web_search", "projects", "workspaces", + "in_region_processing", "none"} +SOURCES = {"detected", "user", "none"} + + +def _read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def _question_sections(path: Path) -> dict[str, str]: + # Same slicing as tests/test_bedrock_migration_assessment_core.py so the parity rules apply unchanged. + text = _read(path) + parts = re.split(r"^## (Q[0-9.]+) — .*$", text, flags=re.M) + sections = {parts[i]: parts[i + 1] for i in range(1, len(parts), 2)} + for qid, body in sections.items(): + sections[qid] = re.split(r"^(?:### Batch|## Step )", body, flags=re.M)[0] + return sections + + +def _interpret_lines(body: str) -> list[str]: + return [line for line in body.splitlines() if line.startswith("Interpret →")] + + +def _table_rows(body: str) -> list[str]: + return [line for line in body.splitlines() if line.startswith("| ") and not line.startswith("| ---")] + + +def _check_entries(entries: list[dict]) -> None: + assert entries, "mantle_requirements.value must carry at least one entry" + for entry in entries: + assert set(entry) <= ENTRY_KEYS, entry + assert {"workload_id", "selector", "source"} <= set(entry), entry + assert entry["selector"] in CLARIFY_SELECTORS, entry + assert entry["source"] in SOURCES, entry + assert ("region" in entry) == (entry["selector"] == "in_region_processing"), entry + + +# --------------------------------------------------------------------------- (a) every clarify file asks the block + +@pytest.mark.parametrize("path", CLARIFY_FILES, ids=lambda p: f"{p.parts[-5]}/{p.name}") +def test_every_clarify_file_asks_the_mantle_only_requirements(path: Path) -> None: + text = _read(path) + assert re.search(r"Mantle-only requirements", text), path + for word in ("asynchronous", "server-side", "Projects", "Workspaces", "Region"): + assert word in text, (path, word) + assert CONSTRAINT in text, path + + +# --------------------------------------------------------------------------- (b) gcp and BMA Q10.5 stay in parity + +def test_q10_5_interpret_line_is_identical_and_names_the_full_vocabulary() -> None: + gcp = _question_sections(GCP_CLARIFY_AI_ONLY) + bma = _question_sections(BMA_CLARIFY_AI_ONLY) + assert list(gcp) == list(bma) + assert list(gcp).index("Q10.5") == list(gcp).index("Q10") + 1 + assert list(gcp).index("Q11") == list(gcp).index("Q10.5") + 1 + gcp_interpret = _interpret_lines(gcp["Q10.5"]) + assert gcp_interpret == _interpret_lines(bma["Q10.5"]) + assert len(gcp_interpret) == 1 + line = gcp_interpret[0] + assert line.startswith(f"Interpret → `{CONSTRAINT}`") + for selector in CLARIFY_SELECTORS: + assert selector in line, selector + assert "`detected | user | none`" in line + assert _table_rows(gcp["Q10.5"]) == _table_rows(bma["Q10.5"]) + assert "_Skip when:_" not in bma["Q10.5"] + assert "_Never skipped:_" in bma["Q10.5"] + assert "mantle_signals_detected" in bma["Q10.5"] + assert "workloads[].mantle_signals" in gcp["Q10.5"] + + +# --------------------------------------------------------------------------- (c) full flow wiring + +def test_full_flow_files_wire_q22b() -> None: + gcp = _read(GCP_CLARIFY_AI) + assert re.search(r"^## Q22b — Mantle-only requirements", gcp, flags=re.M) + assert gcp.index("## Q22 — ") < gcp.index("## Q22b — ") < gcp.index("## Category G") + assert "**Mantle-only signals:**" in gcp + assert f"## Preferences Output — `{CONSTRAINT}`" in gcp + assert '"question_id": "Q22b"' in gcp + assert "5 global questions (Q14, Q15, Q22b, framework, spend)" in gcp + + azure = _read(AZURE_CLARIFY_AI) + assert "**Q22b — Mantle-only requirements.**" in azure + assert '"mantle_requirements": { "disposition": "ESSENTIAL"' in azure + assert f"| `{CONSTRAINT}`" in azure + + assemble = _read(GCP_CLARIFY_ASSEMBLE) + assert "then AI/agentic (Q15, Q22b, Q23–Q25" in assemble + assert "- Q22b (Mantle-only requirements) defaults to the single entry" in assemble + assert '`source: "default:Q22b"`' in assemble + assert "| Q22b — Mantle-only requirements |" in assemble + + +def test_ai_only_fast_paths_list_q10_5() -> None: + gcp = _read(GCP_CLARIFY_AI_ONLY) + assert "Present **only Q1.5, Q2, Q3, Q4, Q10.5, Q11**" in gcp + assert "Q10.5 (Mantle-only requirements) and Q11 (Activate status) are never dropped" in gcp + azure = _read(AZURE_CLARIFY_AI_ONLY) + assert "present ONLY Q1.5, Q2, Q3, Q4, Q10.5, Q11" in azure + assert "**Q10.5 — Mantle-only requirements**" in azure + assert "`mantle_requirements` (Q10.5)" in azure + + +# --------------------------------------------------------------------------- (d) the preferences catalogs know the field + +def test_preferences_catalogs_list_the_field_with_its_question_ids() -> None: + bma = _read(BMA_PREFERENCES_SCHEMA) + table = bma[bma.index("## Constraint catalog"):bma.index("## Validation")] + catalog = {key: (section, qid) for key, section, qid in + re.findall(r"^\| `([a-z_]+)`\s*\| `([a-z_]+)`\s*\| ([^|]+?)\s*\|", table, flags=re.M)} + assert catalog["mantle_requirements"] == ("ai_constraints", "Q10.5") + + gcp = _read(GCP_PREFERENCES_SCHEMA) + table = gcp[gcp.index("## Constraint catalog"):gcp.index("## Report consumption")] + rows = dict(re.findall(r"^\| `([a-z_]+)`\s*\| ([^|]+?)\s*\|", table, flags=re.M)) + assert rows["mantle_requirements"] == "Q22b (AI-only: Q10.5)" + assert "`ai_constraints.mantle_requirements`" in _read(GCP_CLARIFY_AI_ONLY) + + +# --------------------------------------------------------------------------- (e) the BMA run fixture records a valid answer + +def test_bma_run_fixture_records_a_valid_mantle_requirements_answer() -> None: + preferences = json.loads(_read(BMA_RUN_FIXTURE)) + row = preferences["ai_constraints"]["mantle_requirements"] + _check_entries(row["value"]) + assert row["question_id"] == "Q10.5" + assert row["chosen_by"] in {"user", "default"} + assert row["chosen_by"] != "extracted" + if row["chosen_by"] == "default": + assert row["source"] == "default:Q10.5" + assert "Mantle-only requirements were not confirmed by the user" in " ".join( + preferences["metadata"].get("report_caveats", [])) + assert "Q10.5" in preferences["metadata"]["questions_defaulted"] + else: + assert "Q10.5" in preferences["metadata"]["questions_asked"] + + +# --------------------------------------------------------------------------- (f) the azure AI-only example documents the same entry shape + +def _azure_ai_only_example() -> dict: + # Same parse as tests/test_validate_artifacts.py::_ai_only_preferences_example (strip ` //` tails, json-load). + doc = _read(AZURE_CLARIFY_AI_ONLY) + m = re.search(r"## preferences\.json \(AI-only\)\s*```jsonc\n(.*?)\n```", doc, re.S) + lines = [] + for line in m.group(1).split("\n"): + idx = line.find(" //") + lines.append(line[:idx] if idx != -1 else line) + return json.loads("\n".join(lines)) + + +def test_azure_ai_only_example_documents_the_entry_shape() -> None: + row = _azure_ai_only_example()["ai_constraints"]["mantle_requirements"] + _check_entries(row["value"]) + assert row["question_id"] == "Q10.5" + selectors = {entry["selector"] for entry in row["value"]} + assert "in_region_processing" in selectors, "the example must show the `region` key so the shape contract accepts it" + + +# --------------------------------------------------------------------------- (g) Clarify never writes the Design-derived selector + +@pytest.mark.parametrize("path", CLARIFY_FILES, ids=lambda p: f"{p.parts[-5]}/{p.name}") +def test_clarify_leaves_mantle_only_model_or_api_to_design(path: Path) -> None: + for line in _read(path).splitlines(): + if "mantle_only_model_or_api" in line: + assert "Design" in line or "derive" in line, line From 3c452a36050cfaccc0cb8ba2e1dd5f56a176f8ec Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 14:04:06 -0800 Subject: [PATCH 18/41] feat(aws-startup-advisor): derive mantle_selector from ai_constraints.mantle_requirements in Design and disclose the bedrock-runtime assumption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design no longer infers the Bedrock endpoint from the source API, the model or a guess: gcp-to-aws and azure-to-aws design-ai.md gain Step 0.8, which reads preferences.json -> ai_constraints.mantle_requirements (Clarify Q22b / Q10.5), derives bedrock_models[].mantle_selector only from a detected|user entry (plus the catalog for mantle_only_model_or_api), records the cited entry as bedrock_models[].mantle_requirement on every mantle_* row, and writes the verbatim endpoint_assumption sentence ("No Mantle-only requirement was recorded for ; bedrock-runtime selected. ... re-run Clarify.") on every runtime row. Generate echoes whichever field is present in the script/adapter header; the llm-to-bedrock report renders an Endpoint line from a new "Endpoint basis:" context line and cites the public Responses API page for the Runtime background=true 400. Design: skills/gcp-to-aws/references/phases/design/design-ai.md (Step 0.8, Part 5, Part 6 table, Validation Checklist, Present Summary item 6) skills/azure-to-aws/references/phases/design/design-ai.md (Step 0.8, Parts 3/5, Completion gate, Present summary) Schema: skills/shared/ai/schema-design-aws-ai.md (bedrock_models[] bullet, new "mantle_requirement and endpoint_assumption" section, 2 checklist items) + vendored copies in gcp-to-aws/azure-to-aws references/vendored/ai skills/gcp-to-aws/references/shared/schema-design-aws-ai.md skills/azure-to-aws/references/shared/schema-design-aws-ai.md Generate: skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md skills/gcp-to-aws/references/phases/generate/generate-ai.md skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md Report: skills/llm-to-bedrock/SKILL.md (Endpoint policy paragraph, B1 tuple, Endpoint basis: context line) agents/llm2bedrock-report-generator.md (Executive Summary Endpoint line, Model Mapping notes, Limitations) Fixtures: fixtures/aws-design-ai/multi-provider-direct (detected web_search Mantle row + runtime endpoint_assumption), openrouter-gateway-multi- upstream (in_region_processing mantle_requirement + 2 assumptions), new runtime-default-no-requirement (runtime-only design) Tests: tests/test_mantle_requirements_design_contract.py (17 tests) Verification (all run on this commit's tree): - python3 tools/sync-vendored.py && --check: PASS, 64 files (2 re-synced) - python3 scripts/validate-artifacts.py --fixtures: PASS, 101 artifacts, 0 errors, baseline unchanged (MISSING_REQUIRED=6, UNKNOWN_KEY=44) - mise -C plugins/aws-startup-advisor run ci: exit 0 — lint:md 0 errors/823 files, lint:artifacts PASS, lint:vendored-parity PASS, lint:skill-refs PASS (1532 refs), model-id/staleness/catalog-drift/pricing/consent PASS, test 1496 passed + 222 subtests, test:telemetry 311 passed, test:heroku-source-review PASS - skills/llm-to-bedrock/scripts: 655 passed (unchanged) - skills/agent-advisor/scripts: 570 passed (no file under agent-advisor changed) - python3 tools/validate.py: all validations passed; git diff --check clean --- .../agents/llm2bedrock-report-generator.md | 5 +- .../multi-provider-direct/aws-design-ai.json | 12 +- .../aws-design-ai.json | 8 + .../aws-design-ai.json | 77 +++++++ .../references/phases/design/design-ai.md | 58 ++++- .../phases/generate/generate-artifacts-ai.md | 4 + .../references/shared/schema-design-aws-ai.md | 26 ++- .../vendored/ai/schema-design-aws-ai.md | 71 +++++- .../references/phases/design/design-ai.md | 32 ++- .../references/phases/generate/generate-ai.md | 4 +- .../phases/generate/generate-artifacts-ai.md | 9 +- .../references/shared/schema-design-aws-ai.md | 16 ++ .../vendored/ai/schema-design-aws-ai.md | 71 +++++- .../skills/llm-to-bedrock/SKILL.md | 14 +- .../skills/shared/ai/schema-design-aws-ai.md | 71 +++++- ...est_mantle_requirements_design_contract.py | 213 ++++++++++++++++++ 16 files changed, 659 insertions(+), 32 deletions(-) create mode 100644 plugins/aws-startup-advisor/fixtures/aws-design-ai/runtime-default-no-requirement/aws-design-ai.json create mode 100644 plugins/aws-startup-advisor/tests/test_mantle_requirements_design_contract.py diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md index eb902e617..c3ea1a559 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md @@ -752,12 +752,13 @@ banner. Do NOT use the plain ✅ banner just because - **Migration Track:** Track 2 (AI-Only — no infrastructure changes) - **Deliverable:** Git branch `` (the rewriter's returned `branch_name` — may be collision-suffixed, e.g. `bedrock-migration-2`) - **Side-by-side validation:** instead of )` / `No — static baselines only` / `N/A — connectivity only (same-family)` / `N/A — no golden cases`> (matches the banner block chosen above) +- **Endpoint:** , )` per target; when a target is on bedrock-runtime, quote its `endpoint_assumption` sentence verbatim on the next line; when the context line is absent render `bedrock-runtime (plan predates mantle_requirements; re-run Clarify to record a Mantle-only requirement)`> ### Model Mapping | Source Model | Bedrock Model | Notes | | ---------------- | ----------------- | ------- | -| | | | +| | | ; ; )` from the `Endpoint basis:` line; for a runtime target: `bedrock-runtime`> | ### Overall Status: @@ -975,6 +976,8 @@ This evaluation was based on ****: +- **Endpoint choice:** the bedrock-runtime / bedrock-mantle decision above rests on the Mantle-only requirements Clarify recorded (a detected code signal the user confirmed, or the user's own answer) — not on an inspection of every request this app sends. bedrock-runtime rejects `background=true` with a 400 (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, read 2026-11-07); if a workload turns out to need background inference, server-side tools, Projects/Workspaces, or single-Region processing, re-run Clarify rather than editing the generated client. + ### Evaluation methodology disclosure - **Judge model:** ``. The diff --git a/plugins/aws-startup-advisor/fixtures/aws-design-ai/multi-provider-direct/aws-design-ai.json b/plugins/aws-startup-advisor/fixtures/aws-design-ai/multi-provider-direct/aws-design-ai.json index e1abf1aab..41ece30fc 100644 --- a/plugins/aws-startup-advisor/fixtures/aws-design-ai/multi-provider-direct/aws-design-ai.json +++ b/plugins/aws-startup-advisor/fixtures/aws-design-ai/multi-provider-direct/aws-design-ai.json @@ -58,7 +58,7 @@ "target_aws_service": null, "capability": "text_generation", "capability_confidence": "high", - "rationale": "OpenAI-sourced support bot moves to GPT-5.6 Terra on Bedrock Mantle, same provider lineage.", + "rationale": "OpenAI-sourced support bot moves to GPT-5.6 Terra on Bedrock Mantle, same provider lineage; Discover detected the provider-run Web Search tool (src/ai/openai_client.py:14) and the user confirmed it in Clarify, so Mantle is required.", "confidence_warning": null } ], @@ -79,6 +79,7 @@ "migration_complexity": "low", "model_change": true, "migration_path": "direct", + "endpoint_assumption": "No Mantle-only requirement was recorded for wl_anthropic_chat; bedrock-runtime selected. If this workload needs background inference, server-side tools, Projects/Workspaces, or single-Region processing, re-run Clarify.", "quota_risk": "low" }, { @@ -93,7 +94,14 @@ "migration_complexity": "low", "model_change": true, "migration_path": "mantle_openai_responses", - "mantle_selector": "in_region_processing", + "required_client_api": "responses", + "mantle_selector": "web_search", + "mantle_requirement": { + "workload_id": "wl_openai_support_bot", + "selector": "web_search", + "source": "detected", + "evidence": "src/ai/openai_client.py:14 tools=[{\"type\": \"web_search\"}]" + }, "quota_risk": "low" } ], diff --git a/plugins/aws-startup-advisor/fixtures/aws-design-ai/openrouter-gateway-multi-upstream/aws-design-ai.json b/plugins/aws-startup-advisor/fixtures/aws-design-ai/openrouter-gateway-multi-upstream/aws-design-ai.json index 7d03b80a1..9b96a6b8d 100644 --- a/plugins/aws-startup-advisor/fixtures/aws-design-ai/openrouter-gateway-multi-upstream/aws-design-ai.json +++ b/plugins/aws-startup-advisor/fixtures/aws-design-ai/openrouter-gateway-multi-upstream/aws-design-ai.json @@ -129,6 +129,7 @@ "migration_complexity": "medium", "model_change": true, "migration_path": "direct", + "endpoint_assumption": "No Mantle-only requirement was recorded for wl_router_chat_primary; bedrock-runtime selected. If this workload needs background inference, server-side tools, Projects/Workspaces, or single-Region processing, re-run Clarify.", "quota_risk": "low" }, { @@ -143,6 +144,7 @@ "migration_complexity": "medium", "model_change": true, "migration_path": "direct", + "endpoint_assumption": "No Mantle-only requirement was recorded for wl_router_chat_cheap; bedrock-runtime selected. If this workload needs background inference, server-side tools, Projects/Workspaces, or single-Region processing, re-run Clarify.", "quota_risk": "low" }, { @@ -158,6 +160,12 @@ "model_change": true, "migration_path": "mantle_openai_responses", "mantle_selector": "in_region_processing", + "mantle_requirement": { + "workload_id": "wl_router_fallback", + "selector": "in_region_processing", + "source": "user", + "region": "us-east-1" + }, "quota_risk": "medium" } ], diff --git a/plugins/aws-startup-advisor/fixtures/aws-design-ai/runtime-default-no-requirement/aws-design-ai.json b/plugins/aws-startup-advisor/fixtures/aws-design-ai/runtime-default-no-requirement/aws-design-ai.json new file mode 100644 index 000000000..e1024fdb7 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/aws-design-ai/runtime-default-no-requirement/aws-design-ai.json @@ -0,0 +1,77 @@ +{ + "phase": "design", + "focus": "ai", + "timestamp": "2026-11-08T02:20:00Z", + "source_profile": "ai-workload-profile.json", + "metadata": { + "ai_sources": [ + { + "provider": "openai", + "source_service": "openai_api", + "model_families": ["gpt"], + "model_evidence": "observed", + "models": ["gpt-4o"], + "model_observations": [ + { + "normalized_model": "gpt-4o", + "raw_model": "gpt-4o", + "model_family": "gpt", + "evidence": [{"provenance": "code", "confidence": "high", "detail": "src/ai/openai_client.py:9"}] + } + ] + } + ], + "gateways": [], + "bedrock_models_selected": ["us.openai.gpt-5.6-terra"], + "regional_validation": "checked" + }, + "design_blocks": [ + { + "workload_id": "wl_openai_support_bot", + "model_id": "gpt-4o", + "target_bedrock_model": "us.openai.gpt-5.6-terra", + "target_aws_service": null, + "capability": "text_generation", + "capability_confidence": "high", + "rationale": "OpenAI-sourced support bot moves to GPT-5.6 Terra on the bedrock-runtime CRIS path; Clarify recorded no Mantle-only requirement, so the default endpoint applies.", + "confidence_warning": null + } + ], + "ai_architecture": { + "honest_assessment": "strong_migrate", + "honest_assessment_reason": null, + "tiered_strategy": null, + "bedrock_models": [ + { + "source_model_id": "gpt-4o", + "aws_model_id": "us.openai.gpt-5.6-terra", + "capabilities_matched": ["text_generation"], + "capability_gaps": [], + "honest_assessment": "strong_migrate", + "source_provider_price": "2.50/10.00 per 1M tokens", + "bedrock_price": "2.20/13.20 per 1M tokens", + "price_comparison": "mixed", + "migration_complexity": "low", + "model_change": true, + "migration_path": "runtime_openai_cris", + "required_client_api": "responses", + "endpoint_assumption": "No Mantle-only requirement was recorded for wl_openai_support_bot; bedrock-runtime selected. If this workload needs background inference, server-side tools, Projects/Workspaces, or single-Region processing, re-run Clarify.", + "quota_risk": "low" + } + ], + "capability_mapping": { + "text_generation": {"parity": "full", "notes": "The runtime OpenAI-compatible Responses API covers the detected usage."} + }, + "code_migration": { + "primary_pattern": "direct_sdk", + "framework": "none", + "files_to_modify": ["src/ai/openai_client.py"], + "dependency_changes": "openai>=2.45.0 with aws_bedrock_token_generator for the OpenAI client; base URL and model id change only." + }, + "infrastructure": [], + "services_to_migrate": [] + }, + "regional_warnings": [], + "multi_model_warnings": [], + "agentic_design": null +} diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md index 0e11e259c..06e3b9db8 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md @@ -98,6 +98,40 @@ region with geographic `eu.` inference profiles (`global.` forbidden); `pci`/`so CloudTrail on Bedrock API calls + the Part 5 logging lines. Every constraint that changed a choice adds a `regional_warnings[]` entry or a Part 5 line, and the summary names the regime. +## Step 0.8: Endpoint selection from Clarify's Mantle requirements + +Read `preferences.json` → `ai_constraints.mantle_requirements` (Clarify Q22b, or Q10.5 in the +AI-only flow; absent on older runs = no requirement recorded, treat as one +`{ "workload_id": "*", "selector": "none", "source": "none" }` entry). The endpoint of every +`bedrock_models[]` row is decided here and only here — never from the source API, the model +family, or a guess (`references/vendored/ai/bedrock-endpoint-policy.md`). Full field shapes: +`references/shared/schema-design-aws-ai.md` § mantle_selector. + +Per workload (`workloads[]` from `preferences.json`, else the profile; `*` applies to every row): + +1. Match `value[]` entries whose `workload_id` is the workload's id or `*`, whose `selector` is + one of `background_inference | server_side_tools | web_search | projects | workspaces | + in_region_processing`, and whose `source` is `detected` or `user`. `none` matches nothing. +2. **Match → Mantle path for the source API** (`mantle_openai_responses` / `mantle_openai_chat` / + `mantle_messages`) with `mantle_selector` = the matched selector (several → first in the order + above, name the rest in `rationale`) and `mantle_requirement` copied from the entry: + `{ workload_id, selector, source, evidence? (when detected), region? (iff in_region_processing) }`. + `in_region_processing` still obeys the CRIS-only gate in + `references/vendored/ai/ai-openai-to-bedrock.md`: Mantle only when `bedrock-runtime` serves the + model in that `region` through a CRIS profile alone; otherwise runtime in-Region. +3. **`mantle_only_model_or_api`** is never a Clarify value — Design derives it from the catalog + (`ai-openai-to-bedrock.md` / `openai-on-bedrock.md` access paths) and records + `mantle_requirement.source: "catalog"` with the catalog row as `evidence`. +4. **No match → `bedrock-runtime` default** (`runtime_openai_cris` / `runtime_messages` / + `converse` / `gpt-oss` / `direct`) and the row records `endpoint_assumption`, exactly: + `No Mantle-only requirement was recorded for ; bedrock-runtime selected. If this + workload needs background inference, server-side tools, Projects/Workspaces, or single-Region + processing, re-run Clarify.` — `` = the `design_blocks[].workload_id` whose + `model_id` equals the row's `source_model_id` (several → comma-joined). + +Exactly one of `mantle_requirement` / `endpoint_assumption` is present on every row; Generate +echoes it in the script/adapter header and the llm-to-bedrock report quotes it. + ## Part 1: Bedrock model selection **New-target eligibility — before every selection or same-model shortcut, including the @@ -193,7 +227,7 @@ Partial/None. ## Part 3: Integration pattern → effort Map `integration.pattern`: `direct_sdk` (Azure OpenAI/OpenAI, model on Bedrock, region carries -it) → Runtime OpenAI API on `bedrock-runtime` CRIS (Mantle only with a `mantle_selector`), minimal; +it) → Runtime OpenAI API on `bedrock-runtime` CRIS (Mantle only with a `mantle_selector` from Step 0.8), minimal; `direct_sdk` → Bedrock SDK/Converse, medium; `framework` → LangChain/LlamaIndex + Bedrock, low; `rest_api` → Bedrock REST, medium; `mixed` → per-model. @@ -223,8 +257,10 @@ base URL (`https://bedrock-runtime.{region}.amazonaws.com/openai/v1`), credentia Bedrock token or API key), model ID (the CRIS profile), and IAM (`bedrock:InvokeModel*`) change. The **Mantle** path (`migration_path: "mantle_openai_responses"` / `"mantle_openai_chat"`, base URL `.../openai/v1` on `bedrock-mantle`, `bedrock-mantle:*` IAM) is recorded only when the row also -carries `bedrock_models[].mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`); -API continuity alone never selects Mantle. Read `references/shared/openai-on-bedrock.md` for exact +carries `bedrock_models[].mantle_selector` (from Step 0.8 — derived from +`ai_constraints.mantle_requirements` and cited in `mantle_requirement`; +`references/vendored/ai/bedrock-endpoint-policy.md`); API continuity alone never selects Mantle. +Every runtime row carries the Step 0.8 `endpoint_assumption` sentence. Read `references/shared/openai-on-bedrock.md` for exact values. **A Chat Completions source is a supported near-drop-in for GPT-6 and GPT-5.6** (both list Chat Completions on mantle and runtime CRIS per their model cards) — keep `chat.completions.create` unless a Responses-only feature or @@ -283,9 +319,9 @@ deployment names need verified model identity, not a guessed GPT family. Anthropic-SDK sources → keep the Anthropic SDK on the `bedrock-runtime` Messages endpoint: `migration_path: "runtime_messages"` with `required_client_api: "messages"` by default; `"converse"` (boto3 rewrite) only on explicit request or for a Converse-only feature; -`"mantle_messages"` only with a `mantle_selector`. Record `code_migration.openrouter_path` +`"mantle_messages"` only with a `mantle_selector` (from Step 0.8, a `workspaces` entry). Record `code_migration.openrouter_path` when a router was detected (`same_model_runtime|same_model_mantle|direct|litellm|keep_openrouter`; -`same_model_runtime` is the default same-model route, `same_model_mantle` needs a `mantle_selector`). +`same_model_runtime` is the default same-model route, `same_model_mantle` needs a `mantle_selector` from Step 0.8). ## Part 6: Write output @@ -298,7 +334,14 @@ Write `aws-design-ai.json` to `$MIGRATION_DIR/` per `references/shared/schema-de ## Completion gate (fail closed) Before returning control to `design.md`, require `aws-design-ai.json` to exist and pass the -Validation Checklist in `references/shared/schema-design-aws-ai.md`. If it fails: STOP and +Validation Checklist in `references/shared/schema-design-aws-ai.md`, including: + +- Every `mantle_*` row's `mantle_selector` is derived from an `ai_constraints.mantle_requirements` + entry (or the catalog for `mantle_only_model_or_api`) and the row carries `mantle_requirement` + citing it (`selector` equal to `mantle_selector`). +- Every non-`mantle_*` row carries the `endpoint_assumption` sentence naming its workload. + +If it fails: STOP and output "design-ai did not produce a valid aws-design-ai.json; do not complete Phase 3." > **`_assert` has no teeth.** This gate is prose the model both produces and evaluates — CI @@ -309,7 +352,8 @@ output "design-ai did not produce a valid aws-design-ai.json; do not complete Ph Overall honest assessment; the source→Bedrock model comparison (price + assessment per model); integration pattern and complexity; capability gaps; and — prominently, if `weak_migrate`/ -`recommend_stay` — the reason. Name any compliance regime that shaped the design. +`recommend_stay` — the reason. Name any compliance regime that shaped the design. Endpoint per +workload — `bedrock-mantle (, )` or the `endpoint_assumption` sentence verbatim. ## Status — implemented (AI design route) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md index 3823bea19..f3f943622 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -48,6 +48,10 @@ Read `aws-design-ai.json → ai_architecture.code_migration.migration_path` and > adapter. Mantle is used only when Design recorded a `mantle_selector` > (`references/vendored/ai/bedrock-endpoint-policy.md`). The `direct` adapter path is the exception > (cross-family swap, or a Converse feature no same-model path can serve), not the default. +> For a `runtime_openai_cris` or `runtime_messages` row, echo the row's `endpoint_assumption` +> sentence in the generated script/adapter header comment; for a `mantle_*` row, echo that +> selector and its `mantle_requirement` (workload, source, evidence) there instead, so the +> endpoint choice stays traceable to the Clarify answer. ## Step 1M: `migrate_to_mantle.sh` (mantle path) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md index 178fc53cf..597cd9fc5 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md @@ -105,8 +105,9 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` `bedrock_price`, `price_comparison`, `migration_complexity`, `model_change` (bool), `migration_path` (see below), `quota_risk` (`low|medium|high`), `required_client_api` (see below), `mantle_selector` (REQUIRED on every `mantle_*` row, absent otherwise — see - below). Traditional-AI capabilities do NOT appear here — they are `design_blocks[]` rows - with `target_aws_service`. + below), `mantle_requirement` (REQUIRED on every `mantle_*` row) and `endpoint_assumption` + (REQUIRED on every non-`mantle_*` row) — see § mantle_selector. Traditional-AI capabilities do + NOT appear here — they are `design_blocks[]` rows with `target_aws_service`. - **`capability_mapping`** — per capability that is `true` in the profile's `integration.capabilities_summary`: `{ parity: "full|partial|none", notes }`. - **`code_migration`** — `migration_path` (the selected API/endpoint path consumed by Generate), @@ -163,6 +164,23 @@ leaving the `bedrock-runtime` default (definitions in `vendored/ai/bedrock-endpo API continuity (the app already calls Responses, Chat Completions or Messages) is not a selector. Absent on `runtime_*`, `converse`, `gpt-oss` and `direct` rows. +**How it is derived (`mantle_requirement` / `endpoint_assumption`).** `design-ai.md` Step 0.8 +reads `preferences.json → ai_constraints.mantle_requirements.value[]` (Clarify Q22b / Q10.5; +absent on older runs = no requirement recorded). A workload with a matching entry (its own +`workload_id` or `*`) whose `selector` is one of `background_inference | server_side_tools | +web_search | projects | workspaces | in_region_processing` and whose `source` is `detected` or +`user` takes the Mantle path for its source API with that `mantle_selector`, and the row records +`mantle_requirement` — object, REQUIRED on every `mantle_*` row, absent otherwise: +`{ workload_id, selector, source, evidence?, region? }` (`selector` equals `mantle_selector`; +`source` ∈ `detected | user | catalog`; `evidence` present when `source` is `detected` or +`catalog`; `region` present iff `selector` is `in_region_processing`). `mantle_only_model_or_api` +comes from the model catalog with `source: "catalog"`, never from Clarify. Every other row keeps +`bedrock-runtime` and records `endpoint_assumption` — string, REQUIRED on every non-`mantle_*` +row, exactly: `No Mantle-only requirement was recorded for ; bedrock-runtime +selected. If this workload needs background inference, server-side tools, Projects/Workspaces, or +single-Region processing, re-run Clarify.` (`` = the `design_blocks[].workload_id` +whose `model_id` equals the row's `source_model_id`; several → comma-joined). + ## required_client_api (per `bedrock_models[]` entry) `responses | converse | chat_completions | invoke | messages` — records the client API surface the @@ -250,6 +268,10 @@ rubric file (`ai.md`, `ai-azure-openai-to-bedrock.md`, or a `vendored/ai/*` ref) to a Claude/Nova substitute — and records `migration_path: "runtime_openai_cris"` by default. - [ ] Every `bedrock_models[]` row whose `migration_path` starts with `mantle_` carries a `mantle_selector` from the enum above; no `runtime_*`/`converse` row carries one. +- [ ] Every `mantle_*` row carries `mantle_requirement` with `selector == mantle_selector` + (derived in Step 0.8 from `ai_constraints.mantle_requirements`, or the catalog for + `mantle_only_model_or_api`); every non-`mantle_*` row carries the `endpoint_assumption` + sentence naming its workload. - [ ] `regional_warnings` and `multi_model_warnings` are present (`[]` when clean). - [ ] If `agentic_profile.is_agentic == true`: `agentic_design` is present with `migration_approach` matching `preferences.json`; a `harness` source with an diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md index 820592554..250657038 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -178,9 +178,12 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` `migration_path` (see below), `quota_risk` (`low|medium|high`), `required_client_api` (`responses | chat_completions | converse | invoke | messages` — the client API surface the generated code must use; for `runtime_openai_cris` it is the API the source app already - calls, never an implicit Converse), and `mantle_selector` (REQUIRED on every `mantle_*` row, - absent otherwise — see below). Traditional-AI capabilities do NOT appear here — they are - `design_blocks[]` rows with `target_aws_service`. + calls, never an implicit Converse), `mantle_selector` (REQUIRED on every `mantle_*` row, + absent otherwise — see below), `mantle_requirement` (object, REQUIRED on every `mantle_*` row, + absent otherwise — the Clarify entry or catalog row the selector was derived from; see below) + and `endpoint_assumption` (string, REQUIRED on every row whose `migration_path` does NOT start + with `mantle_` — the verbatim bedrock-runtime disclosure; see below). Traditional-AI + capabilities do NOT appear here — they are `design_blocks[]` rows with `target_aws_service`. - **`capability_mapping`** — per capability that is `true` in the profile's `integration.capabilities_summary`: `{ parity: "full|partial|none", notes }`. - **`code_migration`** — `primary_pattern` (matches profile `integration.pattern`), @@ -226,6 +229,61 @@ leaving the `bedrock-runtime` default (definitions in `vendored/ai/bedrock-endpo API continuity (the app already calls Responses, Chat Completions or Messages) is not a selector. Absent on `runtime_*`, `converse`, `gpt-oss` and `direct` rows. +## mantle_requirement and endpoint_assumption (how mantle_selector is derived) + +Design never infers a `mantle_selector` from the source API, the model, or a guess. It reads +`preferences.json → ai_constraints.mantle_requirements.value[]` (written by Clarify — full flow +Q22b, AI-only Q10.5; absent on older runs = no requirement recorded) and applies this rule per +workload: + +- A workload (matched by its own `workload_id`, or by an entry whose `workload_id` is `*`) with an + entry whose `selector` is one of `background_inference | server_side_tools | web_search | + projects | workspaces | in_region_processing` and whose `source` is `detected` or `user` takes + the Mantle path for its source API — `mantle_openai_responses` for a Responses source, + `mantle_openai_chat` for a Chat Completions source, `mantle_messages` for an Anthropic SDK + source — with `mantle_selector` = that selector. Several matching entries → the first in the + policy-table order `background_inference, server_side_tools, web_search, projects, workspaces, + in_region_processing` becomes `mantle_selector`; name the others in the row's `rationale`. + `in_region_processing` applies only when `bedrock-runtime` serves the model in the entry's + `region` through a CRIS profile alone (the CRIS-only gate in `ai-openai-to-bedrock.md`); + otherwise the workload stays on `bedrock-runtime` in that Region. +- `mantle_only_model_or_api` is never a Clarify value: Design derives it from the catalog + (`ai-openai-to-bedrock.md` / `openai-on-bedrock.md` access-path tables) when the selected model + or API exists only on `bedrock-mantle`, and records `mantle_requirement.source: "catalog"`. +- Every other workload keeps the `bedrock-runtime` default and its row records + `endpoint_assumption` verbatim (below). A `none` entry, a default entry, or no entry at all is a + runtime lane by design, not an inference. + +**`bedrock_models[].mantle_requirement`** — object, REQUIRED on every row whose `migration_path` +starts with `mantle_`, absent otherwise: + +```jsonc +{ + "workload_id": "wl_3a1f2c", // the workloads[].workload_id the entry applied to, or "*" + "selector": "web_search", // same value as the row's mantle_selector + "source": "detected", // detected | user | catalog + "evidence": "src/ai/openai_client.py:14 tools=[{\"type\": \"web_search\"}]", // present when source is detected (file:line construct) or catalog (the catalog file/row) + "region": "us-east-1" // present iff selector is in_region_processing +} +``` + +`source: detected | user` entries are copied from the matching +`ai_constraints.mantle_requirements.value[]` entry (same `workload_id`, `selector`, `evidence`, +`region`); `source: "catalog"` is written only by Design for `mantle_only_model_or_api`. + +**`bedrock_models[].endpoint_assumption`** — string, REQUIRED on every row whose +`migration_path` does NOT start with `mantle_` (`runtime_openai_cris`, `runtime_messages`, +`converse`, `gpt-oss`, `direct`), exactly: + +> No Mantle-only requirement was recorded for ``; bedrock-runtime selected. If this +> workload needs background inference, server-side tools, Projects/Workspaces, or single-Region +> processing, re-run Clarify. + +where `` is the `design_blocks[].workload_id` whose `model_id` equals the row's +`source_model_id` (several workloads → comma-joined ids). Generate echoes the sentence in the +generated script/adapter header and the llm-to-bedrock report quotes it, so the user sees that +the runtime choice rests on a recorded answer, not on a silent default. + ## regional_validation `checked` (`aws___get_regional_availability` on the AWS MCP Server confirmed model/region @@ -294,6 +352,13 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` and records `migration_path: "runtime_openai_cris"` by default. - [ ] Every `bedrock_models[]` row whose `migration_path` starts with `mantle_` carries a `mantle_selector` from the enum above; no `runtime_*`/`converse` row carries one. +- [ ] Every `mantle_*` row carries a `mantle_requirement` whose `selector` equals the row's + `mantle_selector`, whose `source` is `detected | user | catalog` (`catalog` iff the selector + is `mantle_only_model_or_api`), and whose `region` is present iff the selector is + `in_region_processing`; the selector was derived from an + `ai_constraints.mantle_requirements` entry (or the catalog), never from the source API. +- [ ] Every non-`mantle_*` row carries the `endpoint_assumption` sentence verbatim, naming the + `design_blocks[].workload_id`(s) whose `model_id` equals the row's `source_model_id`. - [ ] `regional_warnings` and `multi_model_warnings` are present (`[]` when clean). - [ ] If `agentic_profile.is_agentic == true`: `agentic_design` is present with `migration_approach` matching `preferences.json`; a `harness` source whose origin resolves diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index 953869296..2feb1decb 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -27,7 +27,7 @@ Read `$MIGRATION_DIR/preferences.json` → `ai_constraints` (if present). If abs `references/shared/openai-on-bedrock.md` (the fact base: model IDs, endpoint paths, region matrix, quotas, caching). OpenAI's proprietary GPT models run on Bedrock, so the source model is often itself the target — the mapping guide's Tier 0 path is the default, not a cross-family swap. -- `"anthropic"` → load `references/vendored/ai/ai-anthropic-to-bedrock.md` (Anthropic SDK → Bedrock Runtime Messages (default) / Converse alternative; the Anthropic SDK stays and is repointed at `https://bedrock-runtime.{region}.amazonaws.com/anthropic` — record `migration_path: "runtime_messages"` with `required_client_api: "messages"` by default, `"converse"` only on explicit request or for a Converse-only feature, `"mantle_messages"` only with a `mantle_selector`; do NOT use ai-openai-to-bedrock.md for Anthropic SDK users) +- `"anthropic"` → load `references/vendored/ai/ai-anthropic-to-bedrock.md` (Anthropic SDK → Bedrock Runtime Messages (default) / Converse alternative; the Anthropic SDK stays and is repointed at `https://bedrock-runtime.{region}.amazonaws.com/anthropic` — record `migration_path: "runtime_messages"` with `required_client_api: "messages"` by default, `"converse"` only on explicit request or for a Converse-only feature, `"mantle_messages"` only with a `mantle_selector` **recorded by Step 0.8 (a `workspaces` entry in `ai_constraints.mantle_requirements`)**; do NOT use ai-openai-to-bedrock.md for Anthropic SDK users) - `"both"` → `both` never means exactly "Gemini+OpenAI" — per `discover-anthropic-api.md` Step 4.5, `both` is also reached when Anthropic usage joins a Gemini-or-OpenAI codebase, so it can mean "Gemini+Anthropic" @@ -105,6 +105,26 @@ Record what was applied: every constraint that changed a model choice or region --- +## Step 0.8: Endpoint Selection from Clarify's Mantle Requirements + +Read `preferences.json` → `ai_constraints.mantle_requirements` (written by Clarify Q22b, or Q10.5 in the AI-only flow). **Absent on older runs = no requirement recorded** — treat it like a single `{ "workload_id": "*", "selector": "none", "source": "none" }` entry. The endpoint of every `bedrock_models[]` row is decided HERE and only here; the source API, the model family, and your own judgment about what the app "probably" needs are not inputs (`references/vendored/ai/bedrock-endpoint-policy.md`: API continuity alone never selects Mantle). + +**Derivation rule — apply per workload (`workloads[]` from `preferences.json`, falling back to the profile; when the profile has no `workloads[]`, every entry is `*` and applies to every row):** + +1. Collect the `mantle_requirements.value[]` entries whose `workload_id` equals the workload's id or is `*`, and whose `selector` is one of `background_inference | server_side_tools | web_search | projects | workspaces | in_region_processing` with `source` ∈ `detected | user`. Entries with `selector: "none"` or `source: "none"` match nothing. +2. **Any match → the Mantle path for the workload's source API:** `"mantle_openai_responses"` for a Responses source, `"mantle_openai_chat"` for a Chat Completions source, `"mantle_messages"` for an Anthropic SDK source. Set `bedrock_models[].mantle_selector` to the matched selector; when several entries match, take the first in the policy-table order `background_inference, server_side_tools, web_search, projects, workspaces, in_region_processing` and name the others in the row's `rationale`. Copy the matched entry into `bedrock_models[].mantle_requirement` — `{ workload_id, selector, source, evidence?, region? }` where `evidence` is present when `source` is `detected` (the Clarify entry's `file:line construct`) and `region` is present iff `selector` is `in_region_processing`. + - `in_region_processing` still obeys the existing CRIS-only gate in `references/vendored/ai/ai-openai-to-bedrock.md`: it selects Mantle only when `bedrock-runtime` serves the selected model in the entry's `region` through a CRIS profile alone; when a runtime in-Region form exists, the workload stays on `bedrock-runtime` in that Region and the row records the `endpoint_assumption` below with a `rationale` note that the Region need is met on runtime. +3. **`mantle_only_model_or_api` is never a Clarify value.** When the selected model or API exists only on `bedrock-mantle` per the catalog (`references/vendored/ai/ai-openai-to-bedrock.md` / `references/shared/openai-on-bedrock.md` access-path tables), record the Mantle path with `mantle_selector: "mantle_only_model_or_api"` and `mantle_requirement: { "workload_id": "", "selector": "mantle_only_model_or_api", "source": "catalog", "evidence": "" }`. +4. **No match → the `bedrock-runtime` default** (`"runtime_openai_cris"` / `"runtime_messages"` / `"converse"` / `"gpt-oss"` / `"direct"` per Parts 1 and 5) and the row records `bedrock_models[].endpoint_assumption`, exactly: + + > No Mantle-only requirement was recorded for ``; bedrock-runtime selected. If this workload needs background inference, server-side tools, Projects/Workspaces, or single-Region processing, re-run Clarify. + + where `` is the `design_blocks[].workload_id` whose `model_id` equals the row's `source_model_id` (several workloads on one row → comma-joined ids). When the constraint was defaulted (`chosen_by: "default"`), the report caveat "Mantle-only requirements were not confirmed by the user; bedrock-runtime assumed" from Clarify stays in `metadata.report_caveats[]`. + +Exactly one of `mantle_requirement` / `endpoint_assumption` is present on every `bedrock_models[]` row. Generate echoes whichever is present in the generated script/adapter header, and the llm-to-bedrock report quotes it. + +--- + ## Part 1: Bedrock Model Selection **New-target eligibility — before every selection or same-model shortcut, including the @@ -395,7 +415,7 @@ For each detected `integration.pattern` and `ai_source`, generate before/after m | Direct SDK (OpenAI), same model | OpenAI | Selected Mantle API (selector-gated) | Only with `mantle_selector`: base URL → `.../openai/v1` on Mantle, Bedrock credential, `openai.gpt-*` model ID, `bedrock-mantle:*` IAM | | Direct SDK | Vertex AI | boto3 Converse API | `generate_content()` → `converse()` | | Direct SDK | OpenAI | boto3 Converse API | `completions.create()` → `converse()` (use only for a need no same-model path serves, e.g. Knowledge Bases) | -| Direct SDK | Anthropic | Runtime Anthropic Messages (default) | Keep `anthropic.Anthropic`; base URL → `https://bedrock-runtime.{region}.amazonaws.com/anthropic`, string Bedrock token, `global.`/Geo Claude profile ID, `bedrock:InvokeModel*` IAM; `migration_path: "runtime_messages"` + `required_client_api: "messages"`. `"converse"` (boto3 `messages.create()` → `converse()`) only on explicit request / Converse-only feature; `"mantle_messages"` only with a `mantle_selector` | +| Direct SDK | Anthropic | Runtime Anthropic Messages (default) | Keep `anthropic.Anthropic`; base URL → `https://bedrock-runtime.{region}.amazonaws.com/anthropic`, string Bedrock token, `global.`/Geo Claude profile ID, `bedrock:InvokeModel*` IAM; `migration_path: "runtime_messages"` + `required_client_api: "messages"`. `"converse"` (boto3 `messages.create()` → `converse()`) only on explicit request / Converse-only feature; `"mantle_messages"` only with a `mantle_selector` **recorded by Step 0.8 (a `workspaces` entry)** | | LangChain | ChatVertexAI / ChatOpenAI | ChatBedrock | Swap import and model_id | | LlamaIndex | Vertex / OpenAI LLM | BedrockConverse | Swap import | | LLM Router (LiteLLM) | Any | Config change | `model="bedrock/"` (1 line) | @@ -411,7 +431,9 @@ ID becomes the CRIS profile and IAM is `bedrock:InvokeModel*`. A `mantle_*` path (`"mantle_openai_chat"` for a Chat source, `"mantle_openai_responses"` for a Responses source — e.g. Astra on Standard Mantle in `us-east-1` or `us-west-2`) is recorded **only** when the row also records `bedrock_models[].mantle_selector` from `references/vendored/ai/bedrock-endpoint-policy.md` -(the selector enum is defined there and only there); then keep the OpenAI SDK, use +(the selector enum is defined there and only there), derived in Step 0.8 from +`ai_constraints.mantle_requirements` (never from the source API, the model, or a guess) and cited +in the row's `mantle_requirement`; then keep the OpenAI SDK, use `/openai/v1` on Mantle, Bedrock credentials and Mantle IAM. API continuity alone never selects Mantle. Do not reshape Astra Chat solely because it is proprietary GPT. Record `model_change: false` only when the exact source model is retained; a Pro-to-Astra upgrade is a model change requiring eval. @@ -522,7 +544,7 @@ Write `aws-design-ai.json` to `$MIGRATION_DIR/`. | `ai_architecture.honest_assessment` | string | `"strong_migrate"`, `"moderate_migrate"`, `"weak_migrate"`, `"recommend_stay"`, `"not_applicable"` (per-workload only — traditional-AI capabilities `document_extraction`/`image_analysis`/`speech_transcription`; never the overall assessment) | | `ai_architecture.honest_assessment_reason` | string/null | Required when `honest_assessment == "recommend_stay"`. The specific non-cost blocker (region, Realtime API, separate-service capability, Assistants API feature, unsupported API surface). Null otherwise. | | `ai_architecture.tiered_strategy` | object/null | Tiered model routing (null for low/medium volume) | -| `ai_architecture.bedrock_models` | array | Per-model: `gcp_model_id`, `aws_model_id`, `capabilities_matched[]`, `capability_gaps[]`, `honest_assessment`, `source_provider_price`, `bedrock_price`, `price_comparison`, `migration_complexity` | +| `ai_architecture.bedrock_models` | array | Per-model: `gcp_model_id`, `aws_model_id`, `capabilities_matched[]`, `capability_gaps[]`, `honest_assessment`, `source_provider_price`, `bedrock_price`, `price_comparison`, `migration_complexity`, `migration_path`, `mantle_selector` + `mantle_requirement` (every `mantle_*` row, from Step 0.8), `endpoint_assumption` (every non-`mantle_*` row, the Step 0.8 sentence) | | `ai_architecture.capability_mapping` | object | Per-capability: `parity` (full/partial/none), `notes` | | `ai_architecture.code_migration` | object | `primary_pattern`, `framework`, `files_to_modify[]`, `dependency_changes`, `migration_path`; per-model `required_client_api` selects the runtime API | | `ai_architecture.infrastructure` | array | GCP resource → AWS equivalent mappings with confidence | @@ -545,6 +567,7 @@ Write `aws-design-ai.json` to `$MIGRATION_DIR/`. - [ ] If `ai_source` includes `"openai"`: every model that `shared/openai-on-bedrock.md` lists as available on Bedrock, and whose target region carries it, is mapped to **itself** with `model_change: false` — not to a Claude/Nova substitute - [ ] If `honest_assessment == "recommend_stay"`: `honest_assessment_reason` names a non-cost blocker (cost parity alone is never sufficient for an OpenAI source) - [ ] Every same-model proprietary GPT row records `migration_path: "runtime_openai_cris"` by default; every `bedrock_models[]` row whose `migration_path` starts with `mantle_` carries a `mantle_selector` from `references/vendored/ai/bedrock-endpoint-policy.md`, and no `runtime_*`/`converse` row carries one +- [ ] Every `mantle_*` row's `mantle_selector` is derived from an `ai_constraints.mantle_requirements` entry (or the catalog for `mantle_only_model_or_api`) and the row carries `mantle_requirement` citing it; every non-`mantle_*` row carries the `endpoint_assumption` sentence naming its workload - [ ] No `bedrock_models[]` entry pairs a **bare** proprietary GPT id (including bare `openai.gpt-5.5` / `openai.gpt-5.4`) with a Converse/`bedrock-runtime` migration path. GPT-5.6, GPT-6, GPT-5.5 and GPT-5.4 MAY use `migration_path: "runtime_openai_cris"` on @@ -581,3 +604,4 @@ After writing `aws-design-ai.json`, present under 25 lines: 3. Integration pattern and migration complexity 4. Capability gaps (if any) 5. If weak_migrate or recommend_stay: flag prominently with cost justification +6. Endpoint per workload — `bedrock-mantle (, )` or the `endpoint_assumption` sentence verbatim diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md index 0b51bb086..0eddbb6d7 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-ai.md @@ -30,8 +30,8 @@ Check `preferences.json` → `ai_constraints.ai_framework` to determine the migr **OpenAI SDK users keeping the same model (config path for a Responses source; small reshape for a Chat Completions source)** — `ai_framework` = `direct` AND `ai_source` = `openai` AND (`migration_path` == `"runtime_openai_cris"` (the default) OR `migration_path` **starts with** `mantle` (`mantle`, `mantle_openai_responses`, `mantle_openai_chat`, selector-gated)): -- `runtime_openai_cris` (the default — GPT-5.6/GPT-6/GPT-5.5/GPT-5.4 via `bedrock-runtime` CRIS): point the client at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` — **not** the mantle URL. Update the model string to the CRIS inference-profile ID the selected model's card lists (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), not the bare ID. Grant standard `bedrock:InvokeModel*` actions against the CRIS inference-profile ARN — **not** `bedrock-mantle:*`. For every runtime Responses / Chat Completions target — any proprietary GPT CRIS id by id form: GPT-5.6 Sol / Terra / Luna, the GPT-6 family, and GPT-5.5 / GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` alongside the inference-profile ARN: the Responses API authorizes the default project separately from the inference target (, read 2026-11-07) and each model card repeats it for its runtime CRIS ids; `iam_policy.py --target =` (one pair per row, each row's own API) emits this third ARN for exactly the `responses` / `chat_completions` rows — a GPT secondary beside a Claude Messages primary included — and omits it for a `converse` or `messages` row. A source that keeps `client.responses.retrieve(id)` (or `.cancel` / `.delete`) also needs `bedrock:GetInvoke` (`CancelInvoke` / `DeleteInvoke`, only the retained ops) on that same `project/default` ARN — managing a stored response authorizes those actions on the project (same page, "Permissions", read 2026-11-08); `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits them for a runtime `responses` row only, never for a `converse` / `messages` / `chat_completions` row. Read `aws-design-ai.json` → `ai_architecture.bedrock_models[].required_client_api` — the API the source app already calls — and emit that surface (`responses.create`, `chat.completions.create`, or Converse when Design recorded `converse` for a Guardrails or application-inference-profile need); for a region-only move it is simply the source's own API. -- `mantle` / `mantle_openai_responses` / `mantle_openai_chat` (only when Design recorded a `mantle_selector`, see `references/vendored/ai/bedrock-endpoint-policy.md`): point the client at `https://bedrock-mantle.{region}.api.aws/openai/v1` — note the `openai/v1` segment for proprietary GPT models; a bare `/v1` 404s. Swap the credential for a Bedrock API key or the auto-refreshing token provider — **not** the existing OpenAI key. Update the model string to the bare Bedrock model ID (`openai.gpt-*` for a same-model move). Grant the `bedrock-mantle:*` actions; `bedrock:InvokeModel` does not authorize these models. +- `runtime_openai_cris` (the default — GPT-5.6/GPT-6/GPT-5.5/GPT-5.4 via `bedrock-runtime` CRIS): point the client at `https://bedrock-runtime.{region}.amazonaws.com/openai/v1` — **not** the mantle URL. Update the model string to the CRIS inference-profile ID the selected model's card lists (`us.`/`in.`/`global.` for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4), not the bare ID. Grant standard `bedrock:InvokeModel*` actions against the CRIS inference-profile ARN — **not** `bedrock-mantle:*`. For every runtime Responses / Chat Completions target — any proprietary GPT CRIS id by id form: GPT-5.6 Sol / Terra / Luna, the GPT-6 family, and GPT-5.5 / GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default` alongside the inference-profile ARN: the Responses API authorizes the default project separately from the inference target (, read 2026-11-07) and each model card repeats it for its runtime CRIS ids; `iam_policy.py --target =` (one pair per row, each row's own API) emits this third ARN for exactly the `responses` / `chat_completions` rows — a GPT secondary beside a Claude Messages primary included — and omits it for a `converse` or `messages` row. A source that keeps `client.responses.retrieve(id)` (or `.cancel` / `.delete`) also needs `bedrock:GetInvoke` (`CancelInvoke` / `DeleteInvoke`, only the retained ops) on that same `project/default` ARN — managing a stored response authorizes those actions on the project (same page, "Permissions", read 2026-11-08); `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits them for a runtime `responses` row only, never for a `converse` / `messages` / `chat_completions` row. Read `aws-design-ai.json` → `ai_architecture.bedrock_models[].required_client_api` — the API the source app already calls — and emit that surface (`responses.create`, `chat.completions.create`, or Converse when Design recorded `converse` for a Guardrails or application-inference-profile need); for a region-only move it is simply the source's own API. The guide's endpoint line quotes the row's `endpoint_assumption` sentence verbatim, so the reader sees the runtime choice rests on Clarify's recorded answer. +- `mantle` / `mantle_openai_responses` / `mantle_openai_chat` (only when Design recorded a `mantle_selector`, see `references/vendored/ai/bedrock-endpoint-policy.md`): point the client at `https://bedrock-mantle.{region}.api.aws/openai/v1` — note the `openai/v1` segment for proprietary GPT models; a bare `/v1` 404s. Swap the credential for a Bedrock API key or the auto-refreshing token provider — **not** the existing OpenAI key. Update the model string to the bare Bedrock model ID (`openai.gpt-*` for a same-model move). Grant the `bedrock-mantle:*` actions; `bedrock:InvokeModel` does not authorize these models. The guide's endpoint line names the row's `mantle_requirement` (selector, source, evidence) so the Mantle choice stays traceable to the Clarify entry or catalog row. - **Config-only if the source already calls `responses.create`.** A Chat Completions source for GPT-6 or GPT-5.6 may keep `chat.completions.create`/`choices[0].message` parsing unchanged — Chat Completions is supported on both mantle and runtime CRIS for these models, so reshape to Responses is optional (only for a Responses-only feature or demonstrated constraint). GPT-5.5/GPT-5.4 follow the same rule — their model cards (read 2026-11-06) list Chat Completions on both mantle and runtime, so the reshape is optional for them too; probe the selected API against the target account before committing, as for any model. - Test in staging, validate responses diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md index d3ee1f9b0..f97fb2f25 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -56,11 +56,12 @@ Check `preferences.json` → `ai_constraints.ai_framework.value` and `aws-design variant of the same-model script (Step 1M, using the `us.`/`global.`-prefixed inference-profile ID, the `bedrock-runtime` base URL `https://bedrock-runtime.{region}.amazonaws.com/openai/v1`, `bedrock:InvokeModel*` IAM and the row's `required_client_api`) + setup (Step 3) + test harness - (Step 2). Skip provider adapter (Step 1). -- `migration_path` starts with `"mantle"` AND is one of the three OpenAI-SDK spellings — exactly `"mantle"`, `"mantle_openai_chat"` or `"mantle_openai_responses"` (`"mantle_messages"` is NOT this branch; it has its own entry below) → Generate Mantle setup (Step 1M) + setup (Step 3) + test harness (Step 2). Skip provider adapter (Step 1). Design records `"mantle_openai_responses"` or `"mantle_openai_chat"` only together with `bedrock_models[].mantle_selector` (see `references/vendored/ai/bedrock-endpoint-policy.md`); echo that selector in the generated script's header so the Mantle choice stays traceable. Treat the `"mantle_openai_responses"` spelling as this same Mantle path. + (Step 2). Skip provider adapter (Step 1). Echo the row's `endpoint_assumption` sentence in the + generated script's header comment so the bedrock-runtime choice stays traceable to Clarify. +- `migration_path` starts with `"mantle"` AND is one of the three OpenAI-SDK spellings — exactly `"mantle"`, `"mantle_openai_chat"` or `"mantle_openai_responses"` (`"mantle_messages"` is NOT this branch; it has its own entry below) → Generate Mantle setup (Step 1M) + setup (Step 3) + test harness (Step 2). Skip provider adapter (Step 1). Design records `"mantle_openai_responses"` or `"mantle_openai_chat"` only together with `bedrock_models[].mantle_selector` (see `references/vendored/ai/bedrock-endpoint-policy.md`); echo that selector and its `mantle_requirement` (workload, source, evidence) in the generated script's header so the Mantle choice stays traceable. Treat the `"mantle_openai_responses"` spelling as this same Mantle path. - `migration_path` = `"gpt-oss"` → Generate provider adapter (Step 1) targeting the gpt-oss model on Bedrock via the Converse API + setup (Step 3) + test harness (Step 2). Use the Bedrock model ID from `aws-design-ai.json` → `ai_architecture.bedrock_models[].aws_model_id` (the gpt-oss model ID). Do NOT generate a Mantle script — gpt-oss uses the Converse API directly. -- `migration_path` = `"runtime_messages"` (Anthropic source; Design's default for an Anthropic SDK app) → Generate provider adapter (Step 1) whose Bedrock provider class keeps the Anthropic SDK at the runtime base URL `https://bedrock-runtime.{region}.amazonaws.com/anthropic` with a string Bedrock token (`provide_token(region=...)` minted per client) and the row's `global.`/Geo Claude profile ID + setup (Step 3, `bedrock:InvokeModel*` and `bedrock:GetInferenceProfile` IAM) + test harness (Step 2). `required_client_api` is `"messages"`; do NOT generate a Mantle script and do NOT rewrite to Converse. -- `migration_path` = `"mantle_messages"` (Anthropic source, only with `bedrock_models[].mantle_selector`, normally `workspaces`; matched by this entry ONLY, never by the OpenAI-SDK Mantle branch above) → Generate provider adapter (Step 1) with the same Anthropic-SDK adapter shape as `runtime_messages` but at the Mantle base URL `https://bedrock-mantle.{region}.api.aws/anthropic` with the bare `anthropic.claude-*` ID and `bedrock-mantle:*` IAM (Step 3); `required_client_api` stays `"messages"`; echo the selector in the adapter header. Do NOT generate `migrate_to_mantle.sh` (Step 1M) for it. +- `migration_path` = `"runtime_messages"` (Anthropic source; Design's default for an Anthropic SDK app) → Generate provider adapter (Step 1) whose Bedrock provider class keeps the Anthropic SDK at the runtime base URL `https://bedrock-runtime.{region}.amazonaws.com/anthropic` with a string Bedrock token (`provide_token(region=...)` minted per client) and the row's `global.`/Geo Claude profile ID + setup (Step 3, `bedrock:InvokeModel*` and `bedrock:GetInferenceProfile` IAM) + test harness (Step 2). `required_client_api` is `"messages"`; do NOT generate a Mantle script and do NOT rewrite to Converse. Echo the row's `endpoint_assumption` sentence in the generated adapter's header comment. +- `migration_path` = `"mantle_messages"` (Anthropic source, only with `bedrock_models[].mantle_selector`, normally `workspaces`; matched by this entry ONLY, never by the OpenAI-SDK Mantle branch above) → Generate provider adapter (Step 1) with the same Anthropic-SDK adapter shape as `runtime_messages` but at the Mantle base URL `https://bedrock-mantle.{region}.api.aws/anthropic` with the bare `anthropic.claude-*` ID and `bedrock-mantle:*` IAM (Step 3); `required_client_api` stays `"messages"`; echo that selector and its `mantle_requirement` (workload, source, evidence) in the adapter header. Do NOT generate `migrate_to_mantle.sh` (Step 1M) for it. - `"converse"`, `"direct"`, or absent → Generate provider adapter (Step 1) + setup (Step 3) + test harness (Step 2). `"converse"` is an explicit boto3 Converse rewrite (Design records it on request or for a Converse-only feature); do NOT generate a Mantle script for it. - `"llm_router"`, `"api_gateway"`, `"voice_platform"`, or `"framework"` → Skip Step 1, generate gateway config (Step 3B) instead. Step 3B's OpenRouter subroute can still replace that gateway config with Step 1M (`same_model_runtime` → the CRIS variant, `same_model_mantle` → the Mantle variant) or the provider adapter (`direct`); do not decide those from `migration_path` alone. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md index a64fc8b51..ae22c1ba9 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md @@ -95,6 +95,19 @@ same vocabulary as the shared contract (`references/vendored/ai/schema-design-aw endpoint; each such row MUST carry `mantle_selector`, one of `background_inference | server_side_tools | web_search | projects | workspaces | mantle_only_model_or_api | in_region_processing`. API continuity alone is not a selector. +- `bedrock_models[].mantle_requirement` — object, REQUIRED on every `mantle_*` row, absent + otherwise: `{ workload_id, selector, source, evidence?, region? }` — `selector` equals the row's + `mantle_selector`; `source` ∈ `detected | user | catalog` (`detected`/`user` entries are copied + from `preferences.json → ai_constraints.mantle_requirements.value[]`, the Clarify Q22b / Q10.5 + answer `design-ai.md` Step 0.8 reads; `catalog` only for `mantle_only_model_or_api`, derived by + Design from the model catalog); `evidence` present when `source` is `detected` or `catalog`; + `region` present iff `selector` is `in_region_processing`. +- `bedrock_models[].endpoint_assumption` — string, REQUIRED on every row whose `migration_path` + does NOT start with `mantle_`, exactly: `No Mantle-only requirement was recorded for + ; bedrock-runtime selected. If this workload needs background inference, + server-side tools, Projects/Workspaces, or single-Region processing, re-run Clarify.` where + `` is the `design_blocks[].workload_id` whose `model_id` equals the row's + `source_model_id` (several → comma-joined). - `converse`, `gpt-oss`, `direct` — Bedrock-native paths; never paired with a **bare** proprietary `openai.gpt-*` id. - `code_migration.openrouter_path` (router detected) — `same_model_runtime` (default same-model @@ -106,6 +119,9 @@ same vocabulary as the shared contract (`references/vendored/ai/schema-design-aw - [ ] `metadata.ai_source` matches `summary.ai_source` from `ai-workload-profile.json`. - [ ] Every `bedrock_models[]` row whose `migration_path` starts with `mantle_` carries a `mantle_selector`; a same-model proprietary GPT row records `runtime_openai_cris` by default. +- [ ] Every `mantle_*` row carries `mantle_requirement` with `selector == mantle_selector` + (derived from `ai_constraints.mantle_requirements` or the catalog); every non-`mantle_*` + row carries the `endpoint_assumption` sentence naming its workload. - [ ] Every `design_blocks[]` row has exactly one of `target_bedrock_model` / `target_aws_service` non-null (XOR). - [ ] Every traditional-AI row (`document_extraction`/`image_analysis`/`speech_transcription`) diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md index 820592554..250657038 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -178,9 +178,12 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` `migration_path` (see below), `quota_risk` (`low|medium|high`), `required_client_api` (`responses | chat_completions | converse | invoke | messages` — the client API surface the generated code must use; for `runtime_openai_cris` it is the API the source app already - calls, never an implicit Converse), and `mantle_selector` (REQUIRED on every `mantle_*` row, - absent otherwise — see below). Traditional-AI capabilities do NOT appear here — they are - `design_blocks[]` rows with `target_aws_service`. + calls, never an implicit Converse), `mantle_selector` (REQUIRED on every `mantle_*` row, + absent otherwise — see below), `mantle_requirement` (object, REQUIRED on every `mantle_*` row, + absent otherwise — the Clarify entry or catalog row the selector was derived from; see below) + and `endpoint_assumption` (string, REQUIRED on every row whose `migration_path` does NOT start + with `mantle_` — the verbatim bedrock-runtime disclosure; see below). Traditional-AI + capabilities do NOT appear here — they are `design_blocks[]` rows with `target_aws_service`. - **`capability_mapping`** — per capability that is `true` in the profile's `integration.capabilities_summary`: `{ parity: "full|partial|none", notes }`. - **`code_migration`** — `primary_pattern` (matches profile `integration.pattern`), @@ -226,6 +229,61 @@ leaving the `bedrock-runtime` default (definitions in `vendored/ai/bedrock-endpo API continuity (the app already calls Responses, Chat Completions or Messages) is not a selector. Absent on `runtime_*`, `converse`, `gpt-oss` and `direct` rows. +## mantle_requirement and endpoint_assumption (how mantle_selector is derived) + +Design never infers a `mantle_selector` from the source API, the model, or a guess. It reads +`preferences.json → ai_constraints.mantle_requirements.value[]` (written by Clarify — full flow +Q22b, AI-only Q10.5; absent on older runs = no requirement recorded) and applies this rule per +workload: + +- A workload (matched by its own `workload_id`, or by an entry whose `workload_id` is `*`) with an + entry whose `selector` is one of `background_inference | server_side_tools | web_search | + projects | workspaces | in_region_processing` and whose `source` is `detected` or `user` takes + the Mantle path for its source API — `mantle_openai_responses` for a Responses source, + `mantle_openai_chat` for a Chat Completions source, `mantle_messages` for an Anthropic SDK + source — with `mantle_selector` = that selector. Several matching entries → the first in the + policy-table order `background_inference, server_side_tools, web_search, projects, workspaces, + in_region_processing` becomes `mantle_selector`; name the others in the row's `rationale`. + `in_region_processing` applies only when `bedrock-runtime` serves the model in the entry's + `region` through a CRIS profile alone (the CRIS-only gate in `ai-openai-to-bedrock.md`); + otherwise the workload stays on `bedrock-runtime` in that Region. +- `mantle_only_model_or_api` is never a Clarify value: Design derives it from the catalog + (`ai-openai-to-bedrock.md` / `openai-on-bedrock.md` access-path tables) when the selected model + or API exists only on `bedrock-mantle`, and records `mantle_requirement.source: "catalog"`. +- Every other workload keeps the `bedrock-runtime` default and its row records + `endpoint_assumption` verbatim (below). A `none` entry, a default entry, or no entry at all is a + runtime lane by design, not an inference. + +**`bedrock_models[].mantle_requirement`** — object, REQUIRED on every row whose `migration_path` +starts with `mantle_`, absent otherwise: + +```jsonc +{ + "workload_id": "wl_3a1f2c", // the workloads[].workload_id the entry applied to, or "*" + "selector": "web_search", // same value as the row's mantle_selector + "source": "detected", // detected | user | catalog + "evidence": "src/ai/openai_client.py:14 tools=[{\"type\": \"web_search\"}]", // present when source is detected (file:line construct) or catalog (the catalog file/row) + "region": "us-east-1" // present iff selector is in_region_processing +} +``` + +`source: detected | user` entries are copied from the matching +`ai_constraints.mantle_requirements.value[]` entry (same `workload_id`, `selector`, `evidence`, +`region`); `source: "catalog"` is written only by Design for `mantle_only_model_or_api`. + +**`bedrock_models[].endpoint_assumption`** — string, REQUIRED on every row whose +`migration_path` does NOT start with `mantle_` (`runtime_openai_cris`, `runtime_messages`, +`converse`, `gpt-oss`, `direct`), exactly: + +> No Mantle-only requirement was recorded for ``; bedrock-runtime selected. If this +> workload needs background inference, server-side tools, Projects/Workspaces, or single-Region +> processing, re-run Clarify. + +where `` is the `design_blocks[].workload_id` whose `model_id` equals the row's +`source_model_id` (several workloads → comma-joined ids). Generate echoes the sentence in the +generated script/adapter header and the llm-to-bedrock report quotes it, so the user sees that +the runtime choice rests on a recorded answer, not on a silent default. + ## regional_validation `checked` (`aws___get_regional_availability` on the AWS MCP Server confirmed model/region @@ -294,6 +352,13 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` and records `migration_path: "runtime_openai_cris"` by default. - [ ] Every `bedrock_models[]` row whose `migration_path` starts with `mantle_` carries a `mantle_selector` from the enum above; no `runtime_*`/`converse` row carries one. +- [ ] Every `mantle_*` row carries a `mantle_requirement` whose `selector` equals the row's + `mantle_selector`, whose `source` is `detected | user | catalog` (`catalog` iff the selector + is `mantle_only_model_or_api`), and whose `region` is present iff the selector is + `in_region_processing`; the selector was derived from an + `ai_constraints.mantle_requirements` entry (or the catalog), never from the source API. +- [ ] Every non-`mantle_*` row carries the `endpoint_assumption` sentence verbatim, naming the + `design_blocks[].workload_id`(s) whose `model_id` equals the row's `source_model_id`. - [ ] `regional_warnings` and `multi_model_warnings` are present (`[]` when clean). - [ ] If `agentic_profile.is_agentic == true`: `agentic_design` is present with `migration_approach` matching `preferences.json`; a `harness` source whose origin resolves diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md b/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md index 214b5910a..1bd6a7283 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md @@ -509,6 +509,15 @@ continuity alone never selects Mantle — keeping the OpenAI or Anthropic SDK is lane. See `skills/shared/ai/bedrock-endpoint-policy.md` (vendored at `$ASSESS_BASE/references/vendored/ai/bedrock-endpoint-policy.md`) for the public sources. +**How this skill learns a Mantle requirement.** Phase A's delegated Discover records +code-visible selectors as `mantle_signals`; Clarify's Mantle-only requirements block (full flow +Q22b, AI-only Q10.5) confirms them or asks outright and writes +`preferences.json → ai_constraints.mantle_requirements`; Design derives +`bedrock_models[].mantle_selector` only from that field (plus the catalog for +`mantle_only_model_or_api`) and records `mantle_requirement` on Mantle rows and the +`endpoint_assumption` sentence on every runtime row. A row with no recorded requirement is a +runtime lane by design, not an inference. + --- ## Phase A — Assess (runs $ASSESS_SKILL's own AI-path files, inline) @@ -710,7 +719,9 @@ work to the telemetry hooks. Read `$MIGRATION_DIR/aws-design-ai.json` and extract: - `ai_architecture.bedrock_models[]` → array of `{source_model, aws_model_id, use_case, - migration_path, required_client_api, mantle_selector}` (the last three may be absent on older plans) + migration_path, required_client_api, mantle_selector, mantle_requirement, endpoint_assumption}` + (the last five may be absent on older plans; `mantle_requirement` sits on Mantle rows, + `endpoint_assumption` on runtime rows — keep both for the C6 `Endpoint basis:` line) - Collect all `aws_model_id` values into `$TARGET_MODELS` (array). Keep the `use_case` AND the `required_client_api` of each row: B4 passes every row to preflight as its own `=` pair, so a Claude Messages primary and a GPT Chat or Nova @@ -997,6 +1008,7 @@ Resolved target model id: Scripts directory (pinned uv toolchain): <$SCRIPTS> Report date suffix: Usage baseline path: <$BEDROCK_RUN_DIR/usage-baseline.json — C5/C6 dispatches only, omit line if the file is absent> +Endpoint basis: : bedrock-mantle (; ; )" or the row's endpoint_assumption sentence verbatim — C6 dispatch only, omit line when every bedrock_models[] row lacks both fields (older plan)> Source baseline available: Source provider env file: User-supplied log files: diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md index 820592554..250657038 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md @@ -178,9 +178,12 @@ Traditional-AI capabilities (AWS AI service, not Bedrock): `document_extraction` `migration_path` (see below), `quota_risk` (`low|medium|high`), `required_client_api` (`responses | chat_completions | converse | invoke | messages` — the client API surface the generated code must use; for `runtime_openai_cris` it is the API the source app already - calls, never an implicit Converse), and `mantle_selector` (REQUIRED on every `mantle_*` row, - absent otherwise — see below). Traditional-AI capabilities do NOT appear here — they are - `design_blocks[]` rows with `target_aws_service`. + calls, never an implicit Converse), `mantle_selector` (REQUIRED on every `mantle_*` row, + absent otherwise — see below), `mantle_requirement` (object, REQUIRED on every `mantle_*` row, + absent otherwise — the Clarify entry or catalog row the selector was derived from; see below) + and `endpoint_assumption` (string, REQUIRED on every row whose `migration_path` does NOT start + with `mantle_` — the verbatim bedrock-runtime disclosure; see below). Traditional-AI + capabilities do NOT appear here — they are `design_blocks[]` rows with `target_aws_service`. - **`capability_mapping`** — per capability that is `true` in the profile's `integration.capabilities_summary`: `{ parity: "full|partial|none", notes }`. - **`code_migration`** — `primary_pattern` (matches profile `integration.pattern`), @@ -226,6 +229,61 @@ leaving the `bedrock-runtime` default (definitions in `vendored/ai/bedrock-endpo API continuity (the app already calls Responses, Chat Completions or Messages) is not a selector. Absent on `runtime_*`, `converse`, `gpt-oss` and `direct` rows. +## mantle_requirement and endpoint_assumption (how mantle_selector is derived) + +Design never infers a `mantle_selector` from the source API, the model, or a guess. It reads +`preferences.json → ai_constraints.mantle_requirements.value[]` (written by Clarify — full flow +Q22b, AI-only Q10.5; absent on older runs = no requirement recorded) and applies this rule per +workload: + +- A workload (matched by its own `workload_id`, or by an entry whose `workload_id` is `*`) with an + entry whose `selector` is one of `background_inference | server_side_tools | web_search | + projects | workspaces | in_region_processing` and whose `source` is `detected` or `user` takes + the Mantle path for its source API — `mantle_openai_responses` for a Responses source, + `mantle_openai_chat` for a Chat Completions source, `mantle_messages` for an Anthropic SDK + source — with `mantle_selector` = that selector. Several matching entries → the first in the + policy-table order `background_inference, server_side_tools, web_search, projects, workspaces, + in_region_processing` becomes `mantle_selector`; name the others in the row's `rationale`. + `in_region_processing` applies only when `bedrock-runtime` serves the model in the entry's + `region` through a CRIS profile alone (the CRIS-only gate in `ai-openai-to-bedrock.md`); + otherwise the workload stays on `bedrock-runtime` in that Region. +- `mantle_only_model_or_api` is never a Clarify value: Design derives it from the catalog + (`ai-openai-to-bedrock.md` / `openai-on-bedrock.md` access-path tables) when the selected model + or API exists only on `bedrock-mantle`, and records `mantle_requirement.source: "catalog"`. +- Every other workload keeps the `bedrock-runtime` default and its row records + `endpoint_assumption` verbatim (below). A `none` entry, a default entry, or no entry at all is a + runtime lane by design, not an inference. + +**`bedrock_models[].mantle_requirement`** — object, REQUIRED on every row whose `migration_path` +starts with `mantle_`, absent otherwise: + +```jsonc +{ + "workload_id": "wl_3a1f2c", // the workloads[].workload_id the entry applied to, or "*" + "selector": "web_search", // same value as the row's mantle_selector + "source": "detected", // detected | user | catalog + "evidence": "src/ai/openai_client.py:14 tools=[{\"type\": \"web_search\"}]", // present when source is detected (file:line construct) or catalog (the catalog file/row) + "region": "us-east-1" // present iff selector is in_region_processing +} +``` + +`source: detected | user` entries are copied from the matching +`ai_constraints.mantle_requirements.value[]` entry (same `workload_id`, `selector`, `evidence`, +`region`); `source: "catalog"` is written only by Design for `mantle_only_model_or_api`. + +**`bedrock_models[].endpoint_assumption`** — string, REQUIRED on every row whose +`migration_path` does NOT start with `mantle_` (`runtime_openai_cris`, `runtime_messages`, +`converse`, `gpt-oss`, `direct`), exactly: + +> No Mantle-only requirement was recorded for ``; bedrock-runtime selected. If this +> workload needs background inference, server-side tools, Projects/Workspaces, or single-Region +> processing, re-run Clarify. + +where `` is the `design_blocks[].workload_id` whose `model_id` equals the row's +`source_model_id` (several workloads → comma-joined ids). Generate echoes the sentence in the +generated script/adapter header and the llm-to-bedrock report quotes it, so the user sees that +the runtime choice rests on a recorded answer, not on a silent default. + ## regional_validation `checked` (`aws___get_regional_availability` on the AWS MCP Server confirmed model/region @@ -294,6 +352,13 @@ rubric file (`ai.md`, a provider-to-Bedrock mapping guide, or a `vendored/ai/*` and records `migration_path: "runtime_openai_cris"` by default. - [ ] Every `bedrock_models[]` row whose `migration_path` starts with `mantle_` carries a `mantle_selector` from the enum above; no `runtime_*`/`converse` row carries one. +- [ ] Every `mantle_*` row carries a `mantle_requirement` whose `selector` equals the row's + `mantle_selector`, whose `source` is `detected | user | catalog` (`catalog` iff the selector + is `mantle_only_model_or_api`), and whose `region` is present iff the selector is + `in_region_processing`; the selector was derived from an + `ai_constraints.mantle_requirements` entry (or the catalog), never from the source API. +- [ ] Every non-`mantle_*` row carries the `endpoint_assumption` sentence verbatim, naming the + `design_blocks[].workload_id`(s) whose `model_id` equals the row's `source_model_id`. - [ ] `regional_warnings` and `multi_model_warnings` are present (`[]` when clean). - [ ] If `agentic_profile.is_agentic == true`: `agentic_design` is present with `migration_approach` matching `preferences.json`; a `harness` source whose origin resolves diff --git a/plugins/aws-startup-advisor/tests/test_mantle_requirements_design_contract.py b/plugins/aws-startup-advisor/tests/test_mantle_requirements_design_contract.py new file mode 100644 index 000000000..90362a28c --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_mantle_requirements_design_contract.py @@ -0,0 +1,213 @@ +"""Guard: Design derives mantle_selector from Clarify's answer and discloses the runtime assumption. + +PR #585 made `bedrock-runtime` the default endpoint and defined the `mantle_selector` enum, but +nothing told Design where a selector comes from: a plugin reading Terraform or no source code at +all cannot see whether an app needs background inference (`bedrock-runtime` rejects +`background=true` with a 400 — https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, +read 2026-11-07), provider-run tools, Projects/Workspaces or single-Region processing. Discover +now records code-visible signals, Clarify confirms them or asks (Q22b / Q10.5) and writes +`preferences.json → ai_constraints.mantle_requirements`, and Design derives every +`bedrock_models[].mantle_selector` from that field alone (plus the catalog for +`mantle_only_model_or_api`), citing the entry in `mantle_requirement` and writing the verbatim +`endpoint_assumption` sentence on every runtime row. These checks pin the Design/Generate/report +text, the three schema docs and their vendored copies, and the neutral design fixtures. +""" +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +SKILLS = PLUGIN_ROOT / "skills" + +DESIGN_AI_FILES = { + "gcp": SKILLS / "gcp-to-aws" / "references" / "phases" / "design" / "design-ai.md", + "azure": SKILLS / "azure-to-aws" / "references" / "phases" / "design" / "design-ai.md", +} +SCHEMA_DOCS = { + "shared": SKILLS / "shared" / "ai" / "schema-design-aws-ai.md", + "gcp-shadow": SKILLS / "gcp-to-aws" / "references" / "shared" / "schema-design-aws-ai.md", + "azure-shadow": SKILLS / "azure-to-aws" / "references" / "shared" / "schema-design-aws-ai.md", + "gcp-vendored": SKILLS / "gcp-to-aws" / "references" / "vendored" / "ai" / "schema-design-aws-ai.md", + "azure-vendored": SKILLS / "azure-to-aws" / "references" / "vendored" / "ai" / "schema-design-aws-ai.md", +} +GENERATE_FILES = { + "gcp-artifacts": SKILLS / "gcp-to-aws" / "references" / "phases" / "generate" / "generate-artifacts-ai.md", + "azure-artifacts": SKILLS / "azure-to-aws" / "references" / "phases" / "generate" / "generate-artifacts-ai.md", + "gcp-guide": SKILLS / "gcp-to-aws" / "references" / "phases" / "generate" / "generate-ai.md", +} +LLM_TO_BEDROCK_SKILL = SKILLS / "llm-to-bedrock" / "SKILL.md" +REPORT_GENERATOR = PLUGIN_ROOT / "agents" / "llm2bedrock-report-generator.md" +DESIGN_FIXTURES = sorted((PLUGIN_ROOT / "fixtures" / "aws-design-ai").glob("*/aws-design-ai.json")) + +SELECTOR_ENUM = [ + "background_inference", + "server_side_tools", + "web_search", + "projects", + "workspaces", + "mantle_only_model_or_api", + "in_region_processing", +] +ASSUMPTION_PREFIX = "No Mantle-only requirement was recorded for " +ASSUMPTION_SUFFIX = "re-run Clarify." +REQUIREMENT_SOURCES = {"detected", "user", "catalog"} + + +def _read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def _section(text: str, start: str, end: str) -> str: + return text.split(start, 1)[1].split(end, 1)[0] + + +# --------------------------------------------------------------------------- (a) design-ai.md + + +@pytest.mark.parametrize("skill", sorted(DESIGN_AI_FILES)) +def test_design_ai_derives_the_selector_from_clarify_and_discloses_the_runtime_assumption(skill: str) -> None: + text = _read(DESIGN_AI_FILES[skill]) + assert "Step 0.8" in text, f"{skill}: design-ai.md has no Step 0.8 endpoint-selection step" + for token in ( + "ai_constraints.mantle_requirements", + "mantle_requirement", + "endpoint_assumption", + ASSUMPTION_PREFIX.strip(), + "re-run Clarify", + ): + assert token in text, f"{skill}: design-ai.md lacks {token!r}" + step08 = _section(text, "Step 0.8", "## Part 1") + assert "mantle_only_model_or_api" in step08 and "catalog" in step08, ( + f"{skill}: Step 0.8 must say mantle_only_model_or_api is catalog-derived" + ) + assert "in_region_processing" in step08 and "ai-openai-to-bedrock.md" in step08, ( + f"{skill}: Step 0.8 must keep in_region_processing on the CRIS-only gate" + ) + + +# --------------------------------------------------------------------------- (b) schema docs + + +@pytest.mark.parametrize("doc", sorted(SCHEMA_DOCS)) +def test_schema_docs_document_mantle_requirement_and_endpoint_assumption(doc: str) -> None: + text = _read(SCHEMA_DOCS[doc]) + assert "mantle_requirement" in text, f"{doc}: schema doc lacks mantle_requirement" + assert "endpoint_assumption" in text, f"{doc}: schema doc lacks endpoint_assumption" + assert re.search(r"detected\s*\|\s*user\s*\|\s*catalog", text), ( + f"{doc}: schema doc lacks the mantle_requirement.source vocabulary detected | user | catalog" + ) + for value in SELECTOR_ENUM: + assert value in text, f"{doc}: mantle_selector enum value {value} missing" + + +def test_vendored_schema_docs_are_byte_identical_to_the_canonical() -> None: + canonical = SCHEMA_DOCS["shared"].read_bytes() + for key in ("gcp-vendored", "azure-vendored"): + assert SCHEMA_DOCS[key].read_bytes() == canonical, f"{key} drifted — run tools/sync-vendored.py" + + +# --------------------------------------------------------------------------- (c) generate + + +@pytest.mark.parametrize("doc", sorted(GENERATE_FILES)) +def test_generate_echoes_the_requirement_or_the_assumption(doc: str) -> None: + text = _read(GENERATE_FILES[doc]) + assert "endpoint_assumption" in text, f"{doc}: generate file does not echo endpoint_assumption" + assert "mantle_requirement" in text, f"{doc}: generate file does not echo mantle_requirement" + + +# --------------------------------------------------------------------------- (d) llm-to-bedrock + + +def test_llm_to_bedrock_skill_explains_how_a_mantle_requirement_is_learned() -> None: + text = _read(LLM_TO_BEDROCK_SKILL) + policy = _section(text, "## Endpoint policy (bedrock-runtime default)", "## Phase A") + for token in ("ai_constraints.mantle_requirements", "Q22b", "Q10.5", "mantle_signals", "mantle_requirement", "endpoint_assumption"): + assert token in policy, f"llm-to-bedrock Endpoint policy lacks {token!r}" + context = _section(text, "### The context block", "### C0 — Run-context gate") + assert "Endpoint basis:" in context, "context block has no 'Endpoint basis:' line" + assert "C6" in context.split("Endpoint basis:")[1][:400], "Endpoint basis line must scope itself to the C6 dispatch" + b1 = _section(text, "### B1 — Read Assess outputs", "### B2") + assert "mantle_requirement" in b1 and "endpoint_assumption" in b1, "B1 must extract both fields from bedrock_models[]" + + +def test_report_generator_renders_the_endpoint_and_reads_the_endpoint_basis_line() -> None: + text = _read(REPORT_GENERATOR) + assert "**Endpoint:**" in text, "report Executive Summary has no Endpoint line" + assert "Endpoint basis:" in text, "report generator does not read the Endpoint basis: context line" + assert "endpoint_assumption" in text and "mantle_requirement" in text + limitations = _section(text, "## Limitations", "### Evaluation methodology disclosure") + assert "inference-responses-api.html" in limitations and re.search(r"read 20\d\d-\d\d-\d\d", limitations), ( + "Limitations must cite the public Responses API page with a read date" + ) + + +# --------------------------------------------------------------------------- (e) fixtures + + +def _workloads_for(design: dict, source_model_id: str) -> set[str]: + return {b["workload_id"] for b in design["design_blocks"] if b["model_id"] == source_model_id} + + +@pytest.mark.parametrize("fixture", DESIGN_FIXTURES, ids=lambda p: p.parent.name) +def test_fixture_rows_carry_a_requirement_or_the_assumption(fixture: Path) -> None: + design = json.loads(_read(fixture)) + workload_ids = {b["workload_id"] for b in design["design_blocks"]} + rows = design["ai_architecture"]["bedrock_models"] + assert rows, f"{fixture}: no bedrock_models rows" + for row in rows: + label = f"{fixture.parent.name}:{row['aws_model_id']}" + if row["migration_path"].startswith("mantle_"): + assert "endpoint_assumption" not in row, f"{label}: mantle row must not carry endpoint_assumption" + req = row.get("mantle_requirement") + assert isinstance(req, dict), f"{label}: mantle row lacks mantle_requirement" + assert set(req) <= {"workload_id", "selector", "source", "evidence", "region"}, f"{label}: unknown key in mantle_requirement" + assert req["selector"] == row["mantle_selector"], f"{label}: mantle_requirement.selector != mantle_selector" + assert req["source"] in REQUIREMENT_SOURCES, f"{label}: bad mantle_requirement.source" + assert req["workload_id"] in workload_ids | {"*"}, f"{label}: mantle_requirement.workload_id is not a design_blocks id" + if req["source"] in {"detected", "catalog"}: + assert req.get("evidence"), f"{label}: {req['source']} requirement needs evidence" + assert ("region" in req) == (req["selector"] == "in_region_processing"), ( + f"{label}: region must be present iff selector is in_region_processing" + ) + assert (req["source"] == "catalog") == (req["selector"] == "mantle_only_model_or_api"), ( + f"{label}: source catalog iff selector mantle_only_model_or_api" + ) + else: + assert "mantle_requirement" not in row, f"{label}: runtime row must not carry mantle_requirement" + sentence = row.get("endpoint_assumption") + assert isinstance(sentence, str), f"{label}: runtime row lacks endpoint_assumption" + assert sentence.startswith(ASSUMPTION_PREFIX), f"{label}: endpoint_assumption has the wrong prefix" + assert sentence.endswith(ASSUMPTION_SUFFIX), f"{label}: endpoint_assumption has the wrong suffix" + named = sentence[len(ASSUMPTION_PREFIX):].split(";", 1)[0] + named_ids = {w.strip() for w in named.split(",")} + expected = _workloads_for(design, row["source_model_id"]) + assert named_ids and named_ids <= expected, ( + f"{label}: endpoint_assumption names {named_ids}, not a workload whose model_id is {row['source_model_id']!r}" + ) + + +# --------------------------------------------------------------------------- (f) corpus coverage + + +def test_fixture_corpus_covers_a_detected_web_search_row_and_a_runtime_only_design() -> None: + detected_web_search = False + runtime_only = False + for fixture in DESIGN_FIXTURES: + rows = json.loads(_read(fixture))["ai_architecture"]["bedrock_models"] + for row in rows: + req = row.get("mantle_requirement") or {} + if ( + row["migration_path"] == "mantle_openai_responses" + and row.get("mantle_selector") == "web_search" + and req.get("source") == "detected" + ): + detected_web_search = True + if rows and not any(r["migration_path"].startswith("mantle_") for r in rows): + runtime_only = True + assert detected_web_search, "no fixture row is mantle_openai_responses + web_search + source detected" + assert runtime_only, "no fixture has only runtime rows (runtime-default-no-requirement)" From 90b25409f97f3a38105b9221bf9e417d64676bd5 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 14:18:26 -0800 Subject: [PATCH 19/41] fix(aws-startup-advisor): align Mantle selector vocabulary across Discover, Clarify and Design Cross-FEAT integration pass over a6f9ef1a (mantle_signals), 73835b14 (ai_constraints.mantle_requirements) and 922687ea (mantle_requirement / endpoint_assumption): - endpoint_assumption sentence: the shared schema-design-aws-ai.md and the gcp design-ai.md Step 0.8 rendered the placeholder as `` in backticks while the azure docs, the aws-design-ai fixtures and tests/test_mantle_requirements_design_contract.py expect a plain id. The placeholder is now unquoted in both and the "where is" note says the id is written plain (no backticks or quotes). - gcp clarify-ai.md Q22b Preferences Output example: the `evidence` of the detected `projects` entry now copies the Step 5B/5C construct verbatim (src/ai/openai_client.py:9 default_headers={"OpenAI-Project": ...}) so the example matches the "copy the signal's file:line construct" rule. - gcp and azure discover-app-code.md output checklists: the "real line that contains the construct" rule now names the `""` / `line: 0` sentinel that the usage-API Step 4 merges append, matching schema-discover-ai.md. - bedrock-endpoint-policy.md "How a selector is learned": names the Clarify question ids (Q22b / Q10.5) and the Design outputs (mantle_requirement / endpoint_assumption) so the policy describes the whole chain. - Vendored copies re-synced (gcp-to-aws and azure-to-aws references/vendored/ai/bedrock-endpoint-policy.md and schema-design-aws-ai.md). Verification (worktree fix-llm-to-bedrock-runtime-default): - mise -C plugins/aws-startup-advisor run ci: exit 0 (lint:md 0 errors, lint:artifacts PASS 101 artifacts 0 errors, lint:vendored-parity PASS 64, lint:skill-refs PASS 1532, model-id / staleness / catalog-drift / startup-model-pricing / consent-disclosure PASS, test 1496 passed + 222 subtests, test:telemetry 311 passed, test:heroku-source-review PASS). The gitignored local skills/*/scripts/.venv dirs were parked outside the tree for the run (markdownlint and model-id-lint walk into them) and restored. - python3 tools/sync-vendored.py --check: PASS 64 - python3 scripts/validate-artifacts.py --fixtures: PASS 101 artifacts, 0 errors - skills/llm-to-bedrock/scripts: 655 passed - skills/agent-advisor/scripts: 570 passed (no file under agent-advisor changed) - python3 tools/validate.py: All validations passed - git diff --check: clean - mise run build (gitleaks) not run: it fails on pre-existing upstream commit f8d09fd1, unrelated to this branch. --- .../references/phases/discover/discover-app-code.md | 3 ++- .../references/vendored/ai/bedrock-endpoint-policy.md | 7 +++++-- .../references/vendored/ai/schema-design-aws-ai.md | 5 +++-- .../gcp-to-aws/references/phases/clarify/clarify-ai.md | 2 +- .../gcp-to-aws/references/phases/design/design-ai.md | 4 ++-- .../references/phases/discover/discover-app-code.md | 2 +- .../references/vendored/ai/bedrock-endpoint-policy.md | 7 +++++-- .../references/vendored/ai/schema-design-aws-ai.md | 5 +++-- .../skills/shared/ai/bedrock-endpoint-policy.md | 7 +++++-- .../skills/shared/ai/schema-design-aws-ai.md | 5 +++-- 10 files changed, 30 insertions(+), 17 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-app-code.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-app-code.md index ffff9ecc6..e9fb60de4 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-app-code.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-app-code.md @@ -267,7 +267,8 @@ The assembler (`discover-assemble.md`) writes the file; do not update `.phase-st matches `wl_[0-9a-f]{6}`. - [ ] `workloads[].mantle_signals` (optional): `selector` ∈ `background_inference | web_search | server_side_tools | projects | workspaces`; cited lines - are real and contain the `construct`; no secret values. + are real and contain the `construct` (only the usage-API Step 4 merges may append the + `""` / `line: 0` sentinel later); no secret values. - [ ] `integration.gateway_type` set (or null); `capabilities_summary` consistent with `models[].capabilities_used`. - [ ] `is_agentic` cases: `agentic_profile`+`tool_manifest` present iff true; `agent_count` == diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md index ac1d3ca94..6110b3298 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md @@ -63,8 +63,11 @@ The seven values are learned in three different ways; no phase may guess one it profile (bedrock-migration-assessment `discover-app-code.md` Step 4.5); the usage-API discovery files add `projects` / `workspaces` / `web_search` signals from the account's own scoping and model strings. Clarify shows the detected signals and the user confirms or drops each one into - `ai_constraints.mantle_requirements`; when no code was scanned, Clarify asks the same five - outright. A detected signal is a prompt for the question, never the answer. + `ai_constraints.mantle_requirements` (full flow Q22b, AI-only Q10.5); when no code was + scanned, Clarify asks the same five outright. A detected signal is a prompt for the question, + never the answer. Design then derives `bedrock_models[].mantle_selector` from that field alone + and records `mantle_requirement` on every Mantle row and `endpoint_assumption` on every + runtime row (`schema-design-aws-ai.md`). - `background_inference` — positive: `background=True` / `background: true` on `responses.create`, or a raw `"background": true` Responses body. Not a signal: `store=True`, `stream=True`, or polling `responses.retrieve` without `background`. `bedrock-runtime` diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md index 250657038..6d3cc53f4 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -275,12 +275,13 @@ starts with `mantle_`, absent otherwise: `migration_path` does NOT start with `mantle_` (`runtime_openai_cris`, `runtime_messages`, `converse`, `gpt-oss`, `direct`), exactly: -> No Mantle-only requirement was recorded for ``; bedrock-runtime selected. If this +> No Mantle-only requirement was recorded for ; bedrock-runtime selected. If this > workload needs background inference, server-side tools, Projects/Workspaces, or single-Region > processing, re-run Clarify. where `` is the `design_blocks[].workload_id` whose `model_id` equals the row's -`source_model_id` (several workloads → comma-joined ids). Generate echoes the sentence in the +`source_model_id` (several workloads → comma-joined ids), written as a plain id with no +backticks or quotes (`No Mantle-only requirement was recorded for wl_3a1f2c; …`). Generate echoes the sentence in the generated script/adapter header and the llm-to-bedrock report quotes it, so the user sees that the runtime choice rests on a recorded answer, not on a silent default. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai.md index 2b0b6b051..385f7e67f 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai.md @@ -658,7 +658,7 @@ Q22b answers are stored in `preferences.json` → `ai_constraints.mantle_require "ai_constraints": { "mantle_requirements": { "value": [ - { "workload_id": "wl_9532e7", "selector": "projects", "source": "detected", "evidence": "src/ai/openai_client.py:9 default_headers OpenAI-Project" }, + { "workload_id": "wl_9532e7", "selector": "projects", "source": "detected", "evidence": "src/ai/openai_client.py:9 default_headers={\"OpenAI-Project\": os.environ[\"OPENAI_PROJECT_ID\"]}" }, { "workload_id": "*", "selector": "in_region_processing", "source": "user", "region": "us-east-1" } ], "chosen_by": "user", diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index 2feb1decb..53ba58f6d 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -117,9 +117,9 @@ Read `preferences.json` → `ai_constraints.mantle_requirements` (written by Cla 3. **`mantle_only_model_or_api` is never a Clarify value.** When the selected model or API exists only on `bedrock-mantle` per the catalog (`references/vendored/ai/ai-openai-to-bedrock.md` / `references/shared/openai-on-bedrock.md` access-path tables), record the Mantle path with `mantle_selector: "mantle_only_model_or_api"` and `mantle_requirement: { "workload_id": "", "selector": "mantle_only_model_or_api", "source": "catalog", "evidence": "" }`. 4. **No match → the `bedrock-runtime` default** (`"runtime_openai_cris"` / `"runtime_messages"` / `"converse"` / `"gpt-oss"` / `"direct"` per Parts 1 and 5) and the row records `bedrock_models[].endpoint_assumption`, exactly: - > No Mantle-only requirement was recorded for ``; bedrock-runtime selected. If this workload needs background inference, server-side tools, Projects/Workspaces, or single-Region processing, re-run Clarify. + > No Mantle-only requirement was recorded for ; bedrock-runtime selected. If this workload needs background inference, server-side tools, Projects/Workspaces, or single-Region processing, re-run Clarify. - where `` is the `design_blocks[].workload_id` whose `model_id` equals the row's `source_model_id` (several workloads on one row → comma-joined ids). When the constraint was defaulted (`chosen_by: "default"`), the report caveat "Mantle-only requirements were not confirmed by the user; bedrock-runtime assumed" from Clarify stays in `metadata.report_caveats[]`. + where `` is the `design_blocks[].workload_id` whose `model_id` equals the row's `source_model_id` (several workloads on one row → comma-joined ids), written as a plain id with no backticks or quotes. When the constraint was defaulted (`chosen_by: "default"`), the report caveat "Mantle-only requirements were not confirmed by the user; bedrock-runtime assumed" from Clarify stays in `metadata.report_caveats[]`. Exactly one of `mantle_requirement` / `endpoint_assumption` is present on every `bedrock_models[]` row. Generate echoes whichever is present in the generated script/adapter header, and the llm-to-bedrock report quotes it. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-app-code.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-app-code.md index 607e80ad9..395a1f892 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-app-code.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-app-code.md @@ -676,7 +676,7 @@ After generating the output file, the parent `discover.md` handles the phase sta - Every `workloads[].model_id` also appears in `models[].model_id` - `workloads[].capability_confidence` is one of: `"high"`, `"medium"`, `"low"` - `workloads[].workload_id` matches pattern `wl_[0-9a-f]{6}` -- `workloads[].mantle_signals` (optional, Step 5C): every entry has `selector` ∈ `background_inference | web_search | server_side_tools | projects | workspaces`, a repo-relative `file`, a real `line` that contains the quoted `construct`, and no secret value; never `mantle_only_model_or_api` or `in_region_processing` +- `workloads[].mantle_signals` (optional, Step 5C): every entry has `selector` ∈ `background_inference | web_search | server_side_tools | projects | workspaces`, a repo-relative `file`, a real `line` that contains the quoted `construct` (the only exception is the `""` / `line: 0` sentinel that the usage-API Step 4 merges append later), and no secret value; never `mantle_only_model_or_api` or `in_region_processing` - If `is_agentic: true`: `agentic_profile` section exists with all required fields - If `is_agentic: true`: `agentic_profile.agent_count` equals length of `agentic_profile.agents[]` - If `is_agentic: true`: `agentic_profile.tool_count` equals length of `tool_manifest[]` (deduplicated) diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md index ac1d3ca94..6110b3298 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md @@ -63,8 +63,11 @@ The seven values are learned in three different ways; no phase may guess one it profile (bedrock-migration-assessment `discover-app-code.md` Step 4.5); the usage-API discovery files add `projects` / `workspaces` / `web_search` signals from the account's own scoping and model strings. Clarify shows the detected signals and the user confirms or drops each one into - `ai_constraints.mantle_requirements`; when no code was scanned, Clarify asks the same five - outright. A detected signal is a prompt for the question, never the answer. + `ai_constraints.mantle_requirements` (full flow Q22b, AI-only Q10.5); when no code was + scanned, Clarify asks the same five outright. A detected signal is a prompt for the question, + never the answer. Design then derives `bedrock_models[].mantle_selector` from that field alone + and records `mantle_requirement` on every Mantle row and `endpoint_assumption` on every + runtime row (`schema-design-aws-ai.md`). - `background_inference` — positive: `background=True` / `background: true` on `responses.create`, or a raw `"background": true` Responses body. Not a signal: `store=True`, `stream=True`, or polling `responses.retrieve` without `background`. `bedrock-runtime` diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md index 250657038..6d3cc53f4 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -275,12 +275,13 @@ starts with `mantle_`, absent otherwise: `migration_path` does NOT start with `mantle_` (`runtime_openai_cris`, `runtime_messages`, `converse`, `gpt-oss`, `direct`), exactly: -> No Mantle-only requirement was recorded for ``; bedrock-runtime selected. If this +> No Mantle-only requirement was recorded for ; bedrock-runtime selected. If this > workload needs background inference, server-side tools, Projects/Workspaces, or single-Region > processing, re-run Clarify. where `` is the `design_blocks[].workload_id` whose `model_id` equals the row's -`source_model_id` (several workloads → comma-joined ids). Generate echoes the sentence in the +`source_model_id` (several workloads → comma-joined ids), written as a plain id with no +backticks or quotes (`No Mantle-only requirement was recorded for wl_3a1f2c; …`). Generate echoes the sentence in the generated script/adapter header and the llm-to-bedrock report quotes it, so the user sees that the runtime choice rests on a recorded answer, not on a silent default. diff --git a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md index ac1d3ca94..6110b3298 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md @@ -63,8 +63,11 @@ The seven values are learned in three different ways; no phase may guess one it profile (bedrock-migration-assessment `discover-app-code.md` Step 4.5); the usage-API discovery files add `projects` / `workspaces` / `web_search` signals from the account's own scoping and model strings. Clarify shows the detected signals and the user confirms or drops each one into - `ai_constraints.mantle_requirements`; when no code was scanned, Clarify asks the same five - outright. A detected signal is a prompt for the question, never the answer. + `ai_constraints.mantle_requirements` (full flow Q22b, AI-only Q10.5); when no code was + scanned, Clarify asks the same five outright. A detected signal is a prompt for the question, + never the answer. Design then derives `bedrock_models[].mantle_selector` from that field alone + and records `mantle_requirement` on every Mantle row and `endpoint_assumption` on every + runtime row (`schema-design-aws-ai.md`). - `background_inference` — positive: `background=True` / `background: true` on `responses.create`, or a raw `"background": true` Responses body. Not a signal: `store=True`, `stream=True`, or polling `responses.retrieve` without `background`. `bedrock-runtime` diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md index 250657038..6d3cc53f4 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md @@ -275,12 +275,13 @@ starts with `mantle_`, absent otherwise: `migration_path` does NOT start with `mantle_` (`runtime_openai_cris`, `runtime_messages`, `converse`, `gpt-oss`, `direct`), exactly: -> No Mantle-only requirement was recorded for ``; bedrock-runtime selected. If this +> No Mantle-only requirement was recorded for ; bedrock-runtime selected. If this > workload needs background inference, server-side tools, Projects/Workspaces, or single-Region > processing, re-run Clarify. where `` is the `design_blocks[].workload_id` whose `model_id` equals the row's -`source_model_id` (several workloads → comma-joined ids). Generate echoes the sentence in the +`source_model_id` (several workloads → comma-joined ids), written as a plain id with no +backticks or quotes (`No Mantle-only requirement was recorded for wl_3a1f2c; …`). Generate echoes the sentence in the generated script/adapter header and the llm-to-bedrock report quotes it, so the user sees that the runtime choice rests on a recorded answer, not on a silent default. From be3ee4b590af647d14e3647ca10f56de5b904ceb Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 14:36:51 -0800 Subject: [PATCH 20/41] fix(aws-startup-advisor): define Step 0.8 for a Mantle requirement on a source with no Mantle path; widen the Anthropic and report-fallback wording MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round (verdict.json CHANGES_REQUESTED, 1 major + 2 minor): - major — Step 0.8 (gcp and azure design-ai.md) and the derivation section of skills/shared/ai/schema-design-aws-ai.md now define the outcome for a matched ai_constraints.mantle_requirements entry on a workload whose source API has no Mantle path (Gemini / Vertex AI via google-genai, or a workload whose target is a cross-family direct / converse / gpt-oss row): the row keeps bedrock-runtime and the verbatim endpoint_assumption sentence, and discloses the unhonored selector with its source and evidence in rationale and capability_gaps[] (a regional_warnings[] entry for in_region_processing); a Mantle path is never invented and the answer is never dropped. The gcp and azure shadow schema-design-aws-ai.md docs carry the same clause; the Present Summary endpoint line (gcp item 6, azure summary) names the gap. The Clarify copy (gcp Q22b, gcp/BMA Q10.5 identical sentence, azure Q22b bullet and Q10.5 paragraph) tells the user up front that a "yes" for such a workload records a capability gap rather than an endpoint change. tests/test_mantle_requirements_design_contract.py now asserts the Step 0.8 slice names the no-Mantle-path case and capability_gaps. - minor — gcp design-ai.md Step 0 Anthropic bullet and Part 5 Anthropic row, and azure design-ai.md Part 5: "(a `workspaces` entry)" -> "(any matched entry, typically `workspaces`)", matching Step 0.8 and the scanners (Anthropic web_search_20250305 -> web_search, code_execution_* -> server_side_tools). - minor — llm-to-bedrock SKILL.md emits the C6 "Endpoint basis:" line whenever any bedrock_models[] row carries migration_path mantle_*, mantle_selector, mantle_requirement or endpoint_assumption, rendering a Mantle row without mantle_requirement as "bedrock-mantle (; source unknown — plan predates mantle_requirements)"; B1 keeps migration_path / mantle_selector for that purpose. The report generator's Endpoint line renders that form and falls back to the runtime wording only when the line is absent (no row carries any marker). - Vendored copies re-synced (gcp-to-aws and azure-to-aws references/vendored/ai/schema-design-aws-ai.md). Verification (worktree fix-llm-to-bedrock-runtime-default): - mise -C plugins/aws-startup-advisor run ci: exit 0 (lint:md 0 errors, lint:artifacts PASS 101 artifacts 0 errors, lint:vendored-parity PASS 64, lint:skill-refs PASS 1532, model-id / staleness / catalog-drift / startup-model-pricing / consent-disclosure PASS, test 1496 passed + 222 subtests, test:telemetry 311 passed, test:heroku-source-review PASS). The gitignored local skills/*/scripts/.venv dirs were parked outside the tree for the run and restored. - python3 tools/sync-vendored.py --check: PASS 64 - python3 scripts/validate-artifacts.py --fixtures: PASS 101 artifacts, 0 errors - skills/llm-to-bedrock/scripts: 655 passed - skills/agent-advisor/scripts: 570 passed (no file under agent-advisor changed) - python3 tools/validate.py: All validations passed - git diff --check: clean - mise run build (gitleaks) not run: it fails on pre-existing upstream commit f8d09fd1, unrelated to this branch. --- .../agents/llm2bedrock-report-generator.md | 2 +- .../references/phases/clarify/clarify-ai-only.md | 5 ++++- .../references/phases/clarify/clarify-ai.md | 5 ++++- .../azure-to-aws/references/phases/design/design-ai.md | 10 ++++++++-- .../references/shared/schema-design-aws-ai.md | 5 ++++- .../references/vendored/ai/schema-design-aws-ai.md | 6 ++++++ .../references/phases/clarify/clarify-ai-only.md | 2 +- .../references/phases/clarify/clarify-ai-only.md | 2 +- .../gcp-to-aws/references/phases/clarify/clarify-ai.md | 2 +- .../gcp-to-aws/references/phases/design/design-ai.md | 7 ++++--- .../references/shared/schema-design-aws-ai.md | 4 +++- .../references/vendored/ai/schema-design-aws-ai.md | 6 ++++++ .../aws-startup-advisor/skills/llm-to-bedrock/SKILL.md | 6 ++++-- .../skills/shared/ai/schema-design-aws-ai.md | 6 ++++++ .../tests/test_mantle_requirements_design_contract.py | 3 +++ 15 files changed, 56 insertions(+), 15 deletions(-) diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md index c3ea1a559..377f27792 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md @@ -752,7 +752,7 @@ banner. Do NOT use the plain ✅ banner just because - **Migration Track:** Track 2 (AI-Only — no infrastructure changes) - **Deliverable:** Git branch `` (the rewriter's returned `branch_name` — may be collision-suffixed, e.g. `bedrock-migration-2`) - **Side-by-side validation:** instead of )` / `No — static baselines only` / `N/A — connectivity only (same-family)` / `N/A — no golden cases`> (matches the banner block chosen above) -- **Endpoint:** , )` per target; when a target is on bedrock-runtime, quote its `endpoint_assumption` sentence verbatim on the next line; when the context line is absent render `bedrock-runtime (plan predates mantle_requirements; re-run Clarify to record a Mantle-only requirement)`> +- **Endpoint:** , )` per target; when a target is on bedrock-runtime, quote its `endpoint_assumption` sentence verbatim on the next line; a Mantle target the line marks `source unknown — plan predates mantle_requirements` is still rendered as `bedrock-mantle (; source unknown)`; the line is absent only when no row carries a Mantle marker or either field — then render `bedrock-runtime (plan predates mantle_requirements; re-run Clarify to record a Mantle-only requirement)`> ### Model Mapping diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md index fd4daa535..0a200d676 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai-only.md @@ -177,7 +177,10 @@ confirm; otherwise ask. Never dropped. → `ai_constraints.mantle_requirements` server_side_tools | web_search | projects | workspaces | in_region_processing | none`, `source` ∈ `detected | user | none`, `region` required for `in_region_processing`). Default: one `{ "workload_id": "*", "selector": "none", "source": "none" }` entry + report caveat -("Mantle-only requirements were not confirmed by the user; bedrock-runtime assumed"). +("Mantle-only requirements were not confirmed by the user; bedrock-runtime assumed"). For a +workload whose source has no Mantle path (a cross-family `direct` / `converse` / `gpt-oss` +target), say up front that a "yes" does not change the endpoint: Design keeps `bedrock-runtime` +and records the requirement as a capability gap (`rationale` + `capability_gaps[]`). **Q11 — AWS Activate credits** (**never dropped**; ≡ full-flow Q27). Never infer funding stage from Q3 spend. Options: have credits / self-funded (→ Activate Founders up to $5K) / diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md index ec39f6cac..5aabf0f6a 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/clarify/clarify-ai.md @@ -79,7 +79,10 @@ models detected, capabilities, integration pattern + SDK, gateway/router, framew ∈ `detected | user | none`; `region` required for `in_region_processing`). "Use defaults for the rest" writes `[{ "workload_id": "*", "selector": "none", "source": "none" }]` with `chosen_by: "default"`, `source: "default:Q22b"` and the report caveat "Mantle-only - requirements were not confirmed by the user; bedrock-runtime assumed". + requirements were not confirmed by the user; bedrock-runtime assumed". For a workload whose + source has no Mantle path (one Design maps to a cross-family `direct` / `converse` / `gpt-oss` + target), say up front that a "yes" does not change the endpoint: Design keeps `bedrock-runtime` + and records the requirement as a capability gap (`rationale` + `capability_gaps[]`). ### Category G — Agentic (fires ONLY when `agentic_profile.is_agentic == true`) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md index 06e3b9db8..5fb73620b 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md @@ -119,6 +119,11 @@ Per workload (`workloads[]` from `preferences.json`, else the profile; `*` appli `in_region_processing` still obeys the CRIS-only gate in `references/vendored/ai/ai-openai-to-bedrock.md`: Mantle only when `bedrock-runtime` serves the model in that `region` through a CRIS profile alone; otherwise runtime in-Region. + A match on a workload whose source API has no Mantle path (a cross-family `direct` / + `converse` / `gpt-oss` target) does not change the endpoint: the row keeps `bedrock-runtime` + and the verbatim `endpoint_assumption` sentence, and discloses the unhonored selector with its + `source` and `evidence` in `rationale` and `capability_gaps[]` (a `regional_warnings[]` entry + for `in_region_processing`). Never invent a Mantle path for that source; never drop the answer. 3. **`mantle_only_model_or_api`** is never a Clarify value — Design derives it from the catalog (`ai-openai-to-bedrock.md` / `openai-on-bedrock.md` access paths) and records `mantle_requirement.source: "catalog"` with the catalog row as `evidence`. @@ -319,7 +324,7 @@ deployment names need verified model identity, not a guessed GPT family. Anthropic-SDK sources → keep the Anthropic SDK on the `bedrock-runtime` Messages endpoint: `migration_path: "runtime_messages"` with `required_client_api: "messages"` by default; `"converse"` (boto3 rewrite) only on explicit request or for a Converse-only feature; -`"mantle_messages"` only with a `mantle_selector` (from Step 0.8, a `workspaces` entry). Record `code_migration.openrouter_path` +`"mantle_messages"` only with a `mantle_selector` (from Step 0.8 — any matched entry, typically `workspaces`). Record `code_migration.openrouter_path` when a router was detected (`same_model_runtime|same_model_mantle|direct|litellm|keep_openrouter`; `same_model_runtime` is the default same-model route, `same_model_mantle` needs a `mantle_selector` from Step 0.8). @@ -353,7 +358,8 @@ output "design-ai did not produce a valid aws-design-ai.json; do not complete Ph Overall honest assessment; the source→Bedrock model comparison (price + assessment per model); integration pattern and complexity; capability gaps; and — prominently, if `weak_migrate`/ `recommend_stay` — the reason. Name any compliance regime that shaped the design. Endpoint per -workload — `bedrock-mantle (, )` or the `endpoint_assumption` sentence verbatim. +workload — `bedrock-mantle (, )` or the `endpoint_assumption` sentence verbatim, +plus any Step 0.8 requirement that could not be honored (no Mantle path) with its capability gap. ## Status — implemented (AI design route) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md index 597cd9fc5..4a315d67d 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md @@ -174,7 +174,10 @@ web_search | projects | workspaces | in_region_processing` and whose `source` is `{ workload_id, selector, source, evidence?, region? }` (`selector` equals `mantle_selector`; `source` ∈ `detected | user | catalog`; `evidence` present when `source` is `detected` or `catalog`; `region` present iff `selector` is `in_region_processing`). `mantle_only_model_or_api` -comes from the model catalog with `source: "catalog"`, never from Clarify. Every other row keeps +comes from the model catalog with `source: "catalog"`, never from Clarify. A matching entry on a +workload whose source API has no Mantle path (a cross-family `direct` / `converse` / `gpt-oss` +target) keeps `bedrock-runtime` and the verbatim `endpoint_assumption`, and discloses the +unhonored selector in `rationale` and `capability_gaps[]`. Every other row keeps `bedrock-runtime` and records `endpoint_assumption` — string, REQUIRED on every non-`mantle_*` row, exactly: `No Mantle-only requirement was recorded for ; bedrock-runtime selected. If this workload needs background inference, server-side tools, Projects/Workspaces, or diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md index 6d3cc53f4..60376a2ca 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -247,6 +247,12 @@ workload: `in_region_processing` applies only when `bedrock-runtime` serves the model in the entry's `region` through a CRIS profile alone (the CRIS-only gate in `ai-openai-to-bedrock.md`); otherwise the workload stays on `bedrock-runtime` in that Region. +- A matching entry on a workload whose source API has no Mantle path (Gemini / Vertex AI, or a + workload whose target is a cross-family `direct` / `converse` / `gpt-oss` row) does not change + the endpoint: the row keeps `bedrock-runtime`, records `endpoint_assumption` verbatim (below), + and discloses the unhonored selector with its `source` and `evidence` in the row's `rationale` + and `capability_gaps[]` (a `regional_warnings[]` entry for `in_region_processing`). The answer + is never dropped silently and never satisfied by inventing a Mantle path for that source. - `mantle_only_model_or_api` is never a Clarify value: Design derives it from the catalog (`ai-openai-to-bedrock.md` / `openai-on-bedrock.md` access-path tables) when the selected model or API exists only on `bedrock-mantle`, and records `mantle_requirement.source: "catalog"`. diff --git a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md index b42a6a3ec..96d0ced83 100644 --- a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/clarify/clarify-ai-only.md @@ -375,7 +375,7 @@ _Never skipped:_ ask it in Batch 2 unless the user says "use defaults for the re Interpret → `ai_constraints.mantle_requirements`: array of `{ workload_id, selector, source, evidence?, region? }` — `selector` ∈ `background_inference | server_side_tools | web_search | projects | workspaces | in_region_processing | none`, `source` ∈ `detected | user | none`. Default: `[{ "workload_id": "*", "selector": "none", "source": "none" }]` with `chosen_by: "default"` and the report caveat "Mantle-only requirements were not confirmed by the user; bedrock-runtime assumed". -`workload_id` is the `workloads[].workload_id` the answer applies to, or `*` for every workload (always `*` when the profile has no `workloads[]`; the summary's signals are per model, so use `*` unless the user names a workload). `source: "detected"` = prefilled from a summary signal and confirmed by the user (copy the signal's `detail` into `evidence`); `source: "user"` = answered without a signal (an explicit "none" answer records `{ "workload_id": "*", "selector": "none", "source": "user" }`); `source: "none"` = defaulted. `region` is required when `selector` is `in_region_processing`. The constraint wrapper uses `chosen_by: "user"` when answered or confirmed, `chosen_by: "default"` + `source: "default:Q10.5"` when skipped (append the caveat to `metadata.report_caveats[]`) — never `extracted`. `mantle_only_model_or_api` is never written here; Design derives it from the model catalog. +`workload_id` is the `workloads[].workload_id` the answer applies to, or `*` for every workload (always `*` when the profile has no `workloads[]`; the summary's signals are per model, so use `*` unless the user names a workload). `source: "detected"` = prefilled from a summary signal and confirmed by the user (copy the signal's `detail` into `evidence`); `source: "user"` = answered without a signal (an explicit "none" answer records `{ "workload_id": "*", "selector": "none", "source": "user" }`); `source: "none"` = defaulted. `region` is required when `selector` is `in_region_processing`. The constraint wrapper uses `chosen_by: "user"` when answered or confirmed, `chosen_by: "default"` + `source: "default:Q10.5"` when skipped (append the caveat to `metadata.report_caveats[]`) — never `extracted`. `mantle_only_model_or_api` is never written here; Design derives it from the model catalog. For a workload whose source has no Mantle path (Gemini / Vertex AI, or one Design maps to a cross-family `direct` / `converse` / `gpt-oss` target), tell the user up front that a "yes" does not change the endpoint: Design keeps `bedrock-runtime` and records the requirement as a capability gap in the design (`rationale` + `capability_gaps[]`) rather than an endpoint change. ## Q11 — Have you applied for AWS Activate credits? diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md index 0553e6446..352a4f33e 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai-only.md @@ -343,7 +343,7 @@ _Never skipped:_ ask it in Batch 2 unless the user says "use defaults for the re Interpret → `ai_constraints.mantle_requirements`: array of `{ workload_id, selector, source, evidence?, region? }` — `selector` ∈ `background_inference | server_side_tools | web_search | projects | workspaces | in_region_processing | none`, `source` ∈ `detected | user | none`. Default: `[{ "workload_id": "*", "selector": "none", "source": "none" }]` with `chosen_by: "default"` and the report caveat "Mantle-only requirements were not confirmed by the user; bedrock-runtime assumed". -`workload_id` is the `workloads[].workload_id` the answer applies to, or `*` for every workload (always `*` when the profile has no `workloads[]`). `source: "detected"` = prefilled from a Discover signal and confirmed by the user (copy the signal's `file:line construct` into `evidence`); `source: "user"` = answered without a code signal (an explicit "none" answer records `{ "workload_id": "*", "selector": "none", "source": "user" }`); `source: "none"` = defaulted. `region` is required when `selector` is `in_region_processing`. The constraint wrapper uses `chosen_by: "user"` when answered or confirmed, `chosen_by: "default"` + `source: "default:Q10.5"` when skipped (append the caveat to `metadata.report_caveats[]`) — never `extracted`. `mantle_only_model_or_api` is never written here; Design derives it from the model catalog. +`workload_id` is the `workloads[].workload_id` the answer applies to, or `*` for every workload (always `*` when the profile has no `workloads[]`). `source: "detected"` = prefilled from a Discover signal and confirmed by the user (copy the signal's `file:line construct` into `evidence`); `source: "user"` = answered without a code signal (an explicit "none" answer records `{ "workload_id": "*", "selector": "none", "source": "user" }`); `source: "none"` = defaulted. `region` is required when `selector` is `in_region_processing`. The constraint wrapper uses `chosen_by: "user"` when answered or confirmed, `chosen_by: "default"` + `source: "default:Q10.5"` when skipped (append the caveat to `metadata.report_caveats[]`) — never `extracted`. `mantle_only_model_or_api` is never written here; Design derives it from the model catalog. For a workload whose source has no Mantle path (Gemini / Vertex AI, or one Design maps to a cross-family `direct` / `converse` / `gpt-oss` target), tell the user up front that a "yes" does not change the endpoint: Design keeps `bedrock-runtime` and records the requirement as a capability gap in the design (`rationale` + `capability_gaps[]`) rather than an endpoint change. ## Q11 — Have you applied for AWS Activate credits? diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai.md index 385f7e67f..b0058a778 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/clarify/clarify-ai.md @@ -465,7 +465,7 @@ Amazon Bedrock serves most applications on `bedrock-runtime`; a few needs exist Interpret → `ai_constraints.mantle_requirements`: array of `{ workload_id, selector, source, evidence?, region? }` — `selector` ∈ `background_inference | server_side_tools | web_search | projects | workspaces | in_region_processing | none`, `source` ∈ `detected | user | none`. Default: `[{ "workload_id": "*", "selector": "none", "source": "none" }]` with `chosen_by: "default"` and the report caveat "Mantle-only requirements were not confirmed by the user; bedrock-runtime assumed". -`workload_id` is the `workloads[].workload_id` the answer applies to, or `*` for every workload (always `*` when the profile has no `workloads[]`). `source: "detected"` = prefilled from a Discover signal and confirmed by the user (copy the signal's `file:line construct` into `evidence`); `source: "user"` = answered without a code signal (an explicit "none" answer records `{ "workload_id": "*", "selector": "none", "source": "user" }`); `source: "none"` = defaulted. `region` is required when `selector` is `in_region_processing`. The constraint wrapper uses `chosen_by: "user"` when answered or confirmed, `chosen_by: "default"` + `source: "default:Q22b"` when skipped (append the caveat to `metadata.report_caveats[]`) — never `extracted`. `mantle_only_model_or_api` is never written here; Design derives it from the model catalog. +`workload_id` is the `workloads[].workload_id` the answer applies to, or `*` for every workload (always `*` when the profile has no `workloads[]`). `source: "detected"` = prefilled from a Discover signal and confirmed by the user (copy the signal's `file:line construct` into `evidence`); `source: "user"` = answered without a code signal (an explicit "none" answer records `{ "workload_id": "*", "selector": "none", "source": "user" }`); `source: "none"` = defaulted. `region` is required when `selector` is `in_region_processing`. The constraint wrapper uses `chosen_by: "user"` when answered or confirmed, `chosen_by: "default"` + `source: "default:Q22b"` when skipped (append the caveat to `metadata.report_caveats[]`) — never `extracted`. `mantle_only_model_or_api` is never written here; Design derives it from the model catalog. For a workload whose source has no Mantle path (Gemini / Vertex AI, or one Design maps to a cross-family `direct` / `converse` / `gpt-oss` target), tell the user up front that a "yes" does not change the endpoint: Design keeps `bedrock-runtime` and records the requirement as a capability gap in the design (`rationale` + `capability_gaps[]`) rather than an endpoint change. --- diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index 53ba58f6d..1a8b21419 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -27,7 +27,7 @@ Read `$MIGRATION_DIR/preferences.json` → `ai_constraints` (if present). If abs `references/shared/openai-on-bedrock.md` (the fact base: model IDs, endpoint paths, region matrix, quotas, caching). OpenAI's proprietary GPT models run on Bedrock, so the source model is often itself the target — the mapping guide's Tier 0 path is the default, not a cross-family swap. -- `"anthropic"` → load `references/vendored/ai/ai-anthropic-to-bedrock.md` (Anthropic SDK → Bedrock Runtime Messages (default) / Converse alternative; the Anthropic SDK stays and is repointed at `https://bedrock-runtime.{region}.amazonaws.com/anthropic` — record `migration_path: "runtime_messages"` with `required_client_api: "messages"` by default, `"converse"` only on explicit request or for a Converse-only feature, `"mantle_messages"` only with a `mantle_selector` **recorded by Step 0.8 (a `workspaces` entry in `ai_constraints.mantle_requirements`)**; do NOT use ai-openai-to-bedrock.md for Anthropic SDK users) +- `"anthropic"` → load `references/vendored/ai/ai-anthropic-to-bedrock.md` (Anthropic SDK → Bedrock Runtime Messages (default) / Converse alternative; the Anthropic SDK stays and is repointed at `https://bedrock-runtime.{region}.amazonaws.com/anthropic` — record `migration_path: "runtime_messages"` with `required_client_api: "messages"` by default, `"converse"` only on explicit request or for a Converse-only feature, `"mantle_messages"` only with a `mantle_selector` **recorded by Step 0.8 (any matched `ai_constraints.mantle_requirements` entry on the Anthropic workload, typically `workspaces`)**; do NOT use ai-openai-to-bedrock.md for Anthropic SDK users) - `"both"` → `both` never means exactly "Gemini+OpenAI" — per `discover-anthropic-api.md` Step 4.5, `both` is also reached when Anthropic usage joins a Gemini-or-OpenAI codebase, so it can mean "Gemini+Anthropic" @@ -114,6 +114,7 @@ Read `preferences.json` → `ai_constraints.mantle_requirements` (written by Cla 1. Collect the `mantle_requirements.value[]` entries whose `workload_id` equals the workload's id or is `*`, and whose `selector` is one of `background_inference | server_side_tools | web_search | projects | workspaces | in_region_processing` with `source` ∈ `detected | user`. Entries with `selector: "none"` or `source: "none"` match nothing. 2. **Any match → the Mantle path for the workload's source API:** `"mantle_openai_responses"` for a Responses source, `"mantle_openai_chat"` for a Chat Completions source, `"mantle_messages"` for an Anthropic SDK source. Set `bedrock_models[].mantle_selector` to the matched selector; when several entries match, take the first in the policy-table order `background_inference, server_side_tools, web_search, projects, workspaces, in_region_processing` and name the others in the row's `rationale`. Copy the matched entry into `bedrock_models[].mantle_requirement` — `{ workload_id, selector, source, evidence?, region? }` where `evidence` is present when `source` is `detected` (the Clarify entry's `file:line construct`) and `region` is present iff `selector` is `in_region_processing`. - `in_region_processing` still obeys the existing CRIS-only gate in `references/vendored/ai/ai-openai-to-bedrock.md`: it selects Mantle only when `bedrock-runtime` serves the selected model in the entry's `region` through a CRIS profile alone; when a runtime in-Region form exists, the workload stays on `bedrock-runtime` in that Region and the row records the `endpoint_assumption` below with a `rationale` note that the Region need is met on runtime. + - **A match on a workload whose source API has no Mantle path** (Gemini / Vertex AI via `google-genai`, or any workload whose target is a cross-family `"direct"` / `"converse"` / `"gpt-oss"` row) does not change the endpoint. The row keeps its `bedrock-runtime` path and records the `endpoint_assumption` sentence below verbatim (it is the fixed runtime-lane disclosure), and the recorded requirement is disclosed as a capability gap: name the unhonored selector with its `source` and `evidence` in the row's `rationale` and add a `capability_gaps[]` entry (`" requested in Clarify (, ) — no bedrock-mantle path for this source API"`); for `in_region_processing` add a `regional_warnings[]` entry instead. Never invent a Mantle path for that source and never drop the answer silently; the Present Summary endpoint line names the gap. 3. **`mantle_only_model_or_api` is never a Clarify value.** When the selected model or API exists only on `bedrock-mantle` per the catalog (`references/vendored/ai/ai-openai-to-bedrock.md` / `references/shared/openai-on-bedrock.md` access-path tables), record the Mantle path with `mantle_selector: "mantle_only_model_or_api"` and `mantle_requirement: { "workload_id": "", "selector": "mantle_only_model_or_api", "source": "catalog", "evidence": "" }`. 4. **No match → the `bedrock-runtime` default** (`"runtime_openai_cris"` / `"runtime_messages"` / `"converse"` / `"gpt-oss"` / `"direct"` per Parts 1 and 5) and the row records `bedrock_models[].endpoint_assumption`, exactly: @@ -415,7 +416,7 @@ For each detected `integration.pattern` and `ai_source`, generate before/after m | Direct SDK (OpenAI), same model | OpenAI | Selected Mantle API (selector-gated) | Only with `mantle_selector`: base URL → `.../openai/v1` on Mantle, Bedrock credential, `openai.gpt-*` model ID, `bedrock-mantle:*` IAM | | Direct SDK | Vertex AI | boto3 Converse API | `generate_content()` → `converse()` | | Direct SDK | OpenAI | boto3 Converse API | `completions.create()` → `converse()` (use only for a need no same-model path serves, e.g. Knowledge Bases) | -| Direct SDK | Anthropic | Runtime Anthropic Messages (default) | Keep `anthropic.Anthropic`; base URL → `https://bedrock-runtime.{region}.amazonaws.com/anthropic`, string Bedrock token, `global.`/Geo Claude profile ID, `bedrock:InvokeModel*` IAM; `migration_path: "runtime_messages"` + `required_client_api: "messages"`. `"converse"` (boto3 `messages.create()` → `converse()`) only on explicit request / Converse-only feature; `"mantle_messages"` only with a `mantle_selector` **recorded by Step 0.8 (a `workspaces` entry)** | +| Direct SDK | Anthropic | Runtime Anthropic Messages (default) | Keep `anthropic.Anthropic`; base URL → `https://bedrock-runtime.{region}.amazonaws.com/anthropic`, string Bedrock token, `global.`/Geo Claude profile ID, `bedrock:InvokeModel*` IAM; `migration_path: "runtime_messages"` + `required_client_api: "messages"`. `"converse"` (boto3 `messages.create()` → `converse()`) only on explicit request / Converse-only feature; `"mantle_messages"` only with a `mantle_selector` **recorded by Step 0.8 (any matched entry, typically `workspaces`)** | | LangChain | ChatVertexAI / ChatOpenAI | ChatBedrock | Swap import and model_id | | LlamaIndex | Vertex / OpenAI LLM | BedrockConverse | Swap import | | LLM Router (LiteLLM) | Any | Config change | `model="bedrock/"` (1 line) | @@ -604,4 +605,4 @@ After writing `aws-design-ai.json`, present under 25 lines: 3. Integration pattern and migration complexity 4. Capability gaps (if any) 5. If weak_migrate or recommend_stay: flag prominently with cost justification -6. Endpoint per workload — `bedrock-mantle (, )` or the `endpoint_assumption` sentence verbatim +6. Endpoint per workload — `bedrock-mantle (, )` or the `endpoint_assumption` sentence verbatim, plus any Step 0.8 requirement that could not be honored (no Mantle path for the source API) with its capability gap diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md index ae22c1ba9..55b811fde 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md @@ -107,7 +107,9 @@ same vocabulary as the shared contract (`references/vendored/ai/schema-design-aw ; bedrock-runtime selected. If this workload needs background inference, server-side tools, Projects/Workspaces, or single-Region processing, re-run Clarify.` where `` is the `design_blocks[].workload_id` whose `model_id` equals the row's - `source_model_id` (several → comma-joined). + `source_model_id` (several → comma-joined). A Clarify entry on a workload whose source API has + no Mantle path (Gemini / Vertex AI, or a cross-family `direct` / `converse` / `gpt-oss` target) + keeps this sentence and is disclosed in the row's `rationale` and `capability_gaps[]` instead. - `converse`, `gpt-oss`, `direct` — Bedrock-native paths; never paired with a **bare** proprietary `openai.gpt-*` id. - `code_migration.openrouter_path` (router detected) — `same_model_runtime` (default same-model diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md index 6d3cc53f4..60376a2ca 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -247,6 +247,12 @@ workload: `in_region_processing` applies only when `bedrock-runtime` serves the model in the entry's `region` through a CRIS profile alone (the CRIS-only gate in `ai-openai-to-bedrock.md`); otherwise the workload stays on `bedrock-runtime` in that Region. +- A matching entry on a workload whose source API has no Mantle path (Gemini / Vertex AI, or a + workload whose target is a cross-family `direct` / `converse` / `gpt-oss` row) does not change + the endpoint: the row keeps `bedrock-runtime`, records `endpoint_assumption` verbatim (below), + and discloses the unhonored selector with its `source` and `evidence` in the row's `rationale` + and `capability_gaps[]` (a `regional_warnings[]` entry for `in_region_processing`). The answer + is never dropped silently and never satisfied by inventing a Mantle path for that source. - `mantle_only_model_or_api` is never a Clarify value: Design derives it from the catalog (`ai-openai-to-bedrock.md` / `openai-on-bedrock.md` access-path tables) when the selected model or API exists only on `bedrock-mantle`, and records `mantle_requirement.source: "catalog"`. diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md b/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md index 1bd6a7283..08bd80c7c 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/SKILL.md @@ -721,7 +721,9 @@ Read `$MIGRATION_DIR/aws-design-ai.json` and extract: - `ai_architecture.bedrock_models[]` → array of `{source_model, aws_model_id, use_case, migration_path, required_client_api, mantle_selector, mantle_requirement, endpoint_assumption}` (the last five may be absent on older plans; `mantle_requirement` sits on Mantle rows, - `endpoint_assumption` on runtime rows — keep both for the C6 `Endpoint basis:` line) + `endpoint_assumption` on runtime rows — keep both for the C6 `Endpoint basis:` line, and keep + `migration_path` / `mantle_selector` so a Mantle row from a plan that predates + `mantle_requirement` is still reported as Mantle) - Collect all `aws_model_id` values into `$TARGET_MODELS` (array). Keep the `use_case` AND the `required_client_api` of each row: B4 passes every row to preflight as its own `=` pair, so a Claude Messages primary and a GPT Chat or Nova @@ -1008,7 +1010,7 @@ Resolved target model id: Scripts directory (pinned uv toolchain): <$SCRIPTS> Report date suffix: Usage baseline path: <$BEDROCK_RUN_DIR/usage-baseline.json — C5/C6 dispatches only, omit line if the file is absent> -Endpoint basis: : bedrock-mantle (; ; )" or the row's endpoint_assumption sentence verbatim — C6 dispatch only, omit line when every bedrock_models[] row lacks both fields (older plan)> +Endpoint basis: : bedrock-mantle (; ; )" or the row's endpoint_assumption sentence verbatim — C6 dispatch only; a Mantle row (migration_path mantle_* or mantle_selector present) that lacks mantle_requirement renders ": bedrock-mantle (; source unknown — plan predates mantle_requirements)"; emit the line whenever any row carries migration_path mantle_*, mantle_selector, mantle_requirement or endpoint_assumption; omit it only when no row carries any of them (older runtime-only plan)> Source baseline available: Source provider env file: User-supplied log files: diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md index 6d3cc53f4..60376a2ca 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md @@ -247,6 +247,12 @@ workload: `in_region_processing` applies only when `bedrock-runtime` serves the model in the entry's `region` through a CRIS profile alone (the CRIS-only gate in `ai-openai-to-bedrock.md`); otherwise the workload stays on `bedrock-runtime` in that Region. +- A matching entry on a workload whose source API has no Mantle path (Gemini / Vertex AI, or a + workload whose target is a cross-family `direct` / `converse` / `gpt-oss` row) does not change + the endpoint: the row keeps `bedrock-runtime`, records `endpoint_assumption` verbatim (below), + and discloses the unhonored selector with its `source` and `evidence` in the row's `rationale` + and `capability_gaps[]` (a `regional_warnings[]` entry for `in_region_processing`). The answer + is never dropped silently and never satisfied by inventing a Mantle path for that source. - `mantle_only_model_or_api` is never a Clarify value: Design derives it from the catalog (`ai-openai-to-bedrock.md` / `openai-on-bedrock.md` access-path tables) when the selected model or API exists only on `bedrock-mantle`, and records `mantle_requirement.source: "catalog"`. diff --git a/plugins/aws-startup-advisor/tests/test_mantle_requirements_design_contract.py b/plugins/aws-startup-advisor/tests/test_mantle_requirements_design_contract.py index 90362a28c..fbcba6afb 100644 --- a/plugins/aws-startup-advisor/tests/test_mantle_requirements_design_contract.py +++ b/plugins/aws-startup-advisor/tests/test_mantle_requirements_design_contract.py @@ -87,6 +87,9 @@ def test_design_ai_derives_the_selector_from_clarify_and_discloses_the_runtime_a assert "in_region_processing" in step08 and "ai-openai-to-bedrock.md" in step08, ( f"{skill}: Step 0.8 must keep in_region_processing on the CRIS-only gate" ) + assert "no Mantle path" in step08 and "capability_gaps" in step08, ( + f"{skill}: Step 0.8 must define the outcome for a matched entry on a source with no Mantle path" + ) # --------------------------------------------------------------------------- (b) schema docs From 768d75e89f8af4ce6d13d6d57a95a3eb064b53ed Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 15:15:43 -0800 Subject: [PATCH 21/41] fix(aws-startup-advisor): stop quoting one per-token price for both Bedrock endpoints The GPT-5.4 and GPT-5.5 cards publish an In-Region rate only, so a missing Geo or Global row stays unpublished. Co-authored-by: Cursor --- .../references/vendored/ai/bedrock-endpoint-policy.md | 11 ++++++++--- .../references/vendored/ai/bedrock-endpoint-policy.md | 11 ++++++++--- .../skills/shared/ai/bedrock-endpoint-policy.md | 11 ++++++++--- .../tests/test_endpoint_policy_contract.py | 5 +++++ 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md index 6110b3298..eb1331580 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md @@ -4,7 +4,8 @@ > Vendored into each consuming skill as `references/vendored/ai/bedrock-endpoint-policy.md` > and kept byte-identical by `tools/sync-vendored.py`; edit HERE, then run the sync. > Every statement below comes from the public Amazon Bedrock User Guide pages listed in -> § Sources (read 2026-11-07). Re-read those pages before changing a row. +> § Sources (read 2026-11-07; the GPT-5.4 and GPT-5.5 model cards read 2026-11-08). +> Re-read those pages before changing a row. ## The rule @@ -28,8 +29,9 @@ Anthropic Messages source keeps its SDK on `bedrock-runtime` by changing the bas credential and the model ID. Choosing Mantle because "the app already speaks Responses" is not a selector. -**Pricing never selects an endpoint.** Per-token pricing for the same model is identical on both -endpoints; choose by API and capability need. +**Pricing never selects an endpoint.** Choose by API and capability need. Read the per-token +rate from that model's card for the inference option the endpoint serves. A card with no Geo or +Global row has not published that price; do not use the In-Region rate in its place. ## `mantle_selector` — the recorded reason @@ -148,6 +150,9 @@ engine (`endpoints.html`). All pages under `https://docs.aws.amazon.com/bedrock/latest/userguide/`, read 2026-11-07: - `endpoints.html` — https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html +- `model-card-openai-gpt-54.html` and `model-card-openai-gpt-55.html` (read 2026-11-08) — + commercial In-Region prices only. Runtime lists `us.` and `global.` ids, and those rows + publish no price. - `models-endpoint-availability.html` - `inference-responses-api.html` - `inference-chat-completions.html` diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md index 6110b3298..eb1331580 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md @@ -4,7 +4,8 @@ > Vendored into each consuming skill as `references/vendored/ai/bedrock-endpoint-policy.md` > and kept byte-identical by `tools/sync-vendored.py`; edit HERE, then run the sync. > Every statement below comes from the public Amazon Bedrock User Guide pages listed in -> § Sources (read 2026-11-07). Re-read those pages before changing a row. +> § Sources (read 2026-11-07; the GPT-5.4 and GPT-5.5 model cards read 2026-11-08). +> Re-read those pages before changing a row. ## The rule @@ -28,8 +29,9 @@ Anthropic Messages source keeps its SDK on `bedrock-runtime` by changing the bas credential and the model ID. Choosing Mantle because "the app already speaks Responses" is not a selector. -**Pricing never selects an endpoint.** Per-token pricing for the same model is identical on both -endpoints; choose by API and capability need. +**Pricing never selects an endpoint.** Choose by API and capability need. Read the per-token +rate from that model's card for the inference option the endpoint serves. A card with no Geo or +Global row has not published that price; do not use the In-Region rate in its place. ## `mantle_selector` — the recorded reason @@ -148,6 +150,9 @@ engine (`endpoints.html`). All pages under `https://docs.aws.amazon.com/bedrock/latest/userguide/`, read 2026-11-07: - `endpoints.html` — https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html +- `model-card-openai-gpt-54.html` and `model-card-openai-gpt-55.html` (read 2026-11-08) — + commercial In-Region prices only. Runtime lists `us.` and `global.` ids, and those rows + publish no price. - `models-endpoint-availability.html` - `inference-responses-api.html` - `inference-chat-completions.html` diff --git a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md index 6110b3298..eb1331580 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md @@ -4,7 +4,8 @@ > Vendored into each consuming skill as `references/vendored/ai/bedrock-endpoint-policy.md` > and kept byte-identical by `tools/sync-vendored.py`; edit HERE, then run the sync. > Every statement below comes from the public Amazon Bedrock User Guide pages listed in -> § Sources (read 2026-11-07). Re-read those pages before changing a row. +> § Sources (read 2026-11-07; the GPT-5.4 and GPT-5.5 model cards read 2026-11-08). +> Re-read those pages before changing a row. ## The rule @@ -28,8 +29,9 @@ Anthropic Messages source keeps its SDK on `bedrock-runtime` by changing the bas credential and the model ID. Choosing Mantle because "the app already speaks Responses" is not a selector. -**Pricing never selects an endpoint.** Per-token pricing for the same model is identical on both -endpoints; choose by API and capability need. +**Pricing never selects an endpoint.** Choose by API and capability need. Read the per-token +rate from that model's card for the inference option the endpoint serves. A card with no Geo or +Global row has not published that price; do not use the In-Region rate in its place. ## `mantle_selector` — the recorded reason @@ -148,6 +150,9 @@ engine (`endpoints.html`). All pages under `https://docs.aws.amazon.com/bedrock/latest/userguide/`, read 2026-11-07: - `endpoints.html` — https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html +- `model-card-openai-gpt-54.html` and `model-card-openai-gpt-55.html` (read 2026-11-08) — + commercial In-Region prices only. Runtime lists `us.` and `global.` ids, and those rows + publish no price. - `models-endpoint-availability.html` - `inference-responses-api.html` - `inference-chat-completions.html` diff --git a/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py b/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py index 45c38073e..9b2633d5a 100644 --- a/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py +++ b/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py @@ -68,6 +68,11 @@ def test_policy_states_the_runtime_default_and_selector_rule() -> None: text = _norm(POLICY_CANONICAL.read_text(encoding="utf-8")) assert "Use `bedrock-runtime` for new applications" in text assert "API continuity alone never selects Mantle" in text + assert "Pricing never selects an endpoint" in text + assert "identical on both" not in text + assert "do not use the In-Region rate in its place" in text + assert "model-card-openai-gpt-54.html" in text + assert "model-card-openai-gpt-55.html" in text for value in MANTLE_SELECTOR_ENUM: assert f"`{value}`" in text, f"mantle_selector enum value {value} missing from the policy" assert ENDPOINTS_URL in text, "policy must cite the public endpoints.html page" From c23d9072ff81bf6d52419a6756a942d3bbbb1950 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 9 Nov 2026 09:08:38 -0800 Subject: [PATCH 22/41] fix(aws-startup-advisor): iam_policy dispatches stored-response grants by endpoint and emits Web Search grants for a web_search row (PR 588 F12, F15) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F12 — `generate_policy` now splits `responses` rows by endpoint: a runtime GPT CRIS row keeps `bedrock:GetInvoke` / `CancelInvoke` / `DeleteInvoke` on `project/default`; a bare Mantle id on `responses` is covered by the `BedrockMantleInference` statement (Mantle inference, stored-response retrieval at `GET /v1/responses/{id}` included, authorizes `bedrock-mantle:CreateInference`) and no longer raises `stored_response_ops_no_responses_row`. Converse / Messages / Chat / embedding / id-only plans still fail closed. Public basis: inference-responses-api.html "Permissions", read 2026-11-09. F15 — new `--mantle-selector` (repeatable; `parse_mantle_selectors` fails closed with reason `unknown_mantle_selector`) and `--external-web-access`. A `web_search` selector with a Mantle row appends `BedrockWebSearch`: `bedrock-websearch:InvokeSearch` + `InvokeFetch` on `*` with an `aws:RequestedRegion` condition (the service's only condition key; all three actions take `*`), plus `bedrock-websearch:ExternalWebAccess` only when the user confirmed external access. `web_search` without a Mantle row → `web_search_without_mantle_row`; `--external-web-access` without `web_search` → `external_web_access_without_web_search`. Other selectors add nothing. Public basis: web-search.html, security-web-search.html, read 2026-11-09 (`external_web_access` defaults to true; without the action every Fetch fails authorization while the request still returns 200). Recipes: report §6.4 command block gains both flags; the stored-response paragraph keeps its pinned sentences and adds the Mantle dispatch; new "Web Search needs its own grants" paragraph; How-to-Apply gains the Mantle create-plus-retrieve variant and a conditional "Tool-level permission check" bullet. GCP / Azure generate-artifacts-ai.md and the canonical OpenAI guide (+ vendored copies via sync-vendored) carry the same facts and a Step-5 tool-level check. Tests: test_iam_policy_round2.py — TestStoredResponseOps replaces the bare-Mantle raise test with all-Mantle retained-retrieve (Mantle policy, no GetInvoke, legacy form too), runtime control, mixed runtime+Mantle (GetInvoke once + Mantle sids), Mantle Chat-only still fails closed, CLI all-Mantle exits 0; new TestWebSearchGrants (grant shape, external action, parametrized absence, runtime-only raise, unknown selector, CLI emit / refuse / reasons). New tests/test_runtime_default_review_round3.py pins the recipe strings. Validation: skills/llm-to-bedrock/scripts `uv run --group dev pytest -q` 672 passed (was 655); plugin `pytest tests -q` 1532 passed, 1 failed (known test_model_id_lint::test_real_plugin_passes local .venv noise, pre-existing); tools/sync-vendored.py --check PASS (64); markdownlint-cli2 0 issues. --- .../agents/llm2bedrock-report-generator.md | 49 +++- .../phases/generate/generate-artifacts-ai.md | 29 ++- .../vendored/ai/ai-openai-to-bedrock.md | 15 +- .../phases/generate/generate-artifacts-ai.md | 4 +- .../vendored/ai/ai-openai-to-bedrock.md | 15 +- .../llm-to-bedrock/scripts/iam_policy.py | 217 +++++++++++++++--- .../scripts/test_iam_policy_round2.py | 134 ++++++++++- .../skills/shared/ai/ai-openai-to-bedrock.md | 15 +- .../test_runtime_default_review_round3.py | 175 ++++++++++++++ 9 files changed, 594 insertions(+), 59 deletions(-) create mode 100644 plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md index 377f27792..613df593c 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md @@ -567,6 +567,8 @@ Generate a least-privilege IAM policy scoped to the exact model ARNs selected du uv run --project python /iam_policy.py \ --target = [--target ... one per bedrock_models[] row] \ [--stored-response-ops ] \ + [--mantle-selector ] \ + [--external-web-access — ONLY when the user explicitly confirmed external web access for a web_search row] \ --region \ --account-id \ --migration-path \ @@ -616,12 +618,35 @@ on `arn:aws:bedrock:::project/default` with ONLY the retained a project, while retrieving, canceling or deleting a stored response authorizes those three actions on the project (, "Permissions", read 2026-11-08). Without it, creation and the preflight ping pass while the app's -existing `responses.retrieve(id)` is denied. Stored responses exist only on the runtime Responses API: the -helper emits the statement only when a `--target` row is a runtime GPT CRIS id on `responses`, and exits 1 -with `reason: stored_response_ops_no_responses_row` for a Converse / Anthropic Messages / Chat Completions -plan — report that verbatim (the analyzer should not have recorded ops for such a source) and do not add the -actions by hand. When the field is absent or `[]`, omit the flag; the policy then carries no `GetInvoke` / -`CancelInvoke` / `DeleteInvoke`. +existing `responses.retrieve(id)` is denied. Stored responses exist on the Responses API of either endpoint +and the grant is dispatched by endpoint: the helper emits the `BedrockStoredResponseOps` statement only when +a `--target` row is a runtime GPT CRIS id on `responses`; a `--target` row that is a bare Mantle id on +`responses` (a `mantle_openai_responses` plan) satisfies retained ops through the `BedrockMantleInference` +statement — stored responses are retrievable on Mantle at `GET /v1/responses/{id}` and Mantle inference +authorizes `bedrock-mantle:CreateInference` (same page, "Permissions", read 2026-11-09) — so no +`bedrock:GetInvoke` is added for it and the helper does not exit 1. The helper exits 1 with +`reason: stored_response_ops_no_responses_row` for a Converse / Anthropic Messages / Chat Completions +plan, which has no Responses row on either endpoint — report that verbatim (the analyzer should not have +recorded ops for such a source) and do not add the actions by hand. When the field is absent or `[]`, omit +the flag; the policy then carries no `GetInvoke` / `CancelInvoke` / `DeleteInvoke`. + +**Web Search needs its own grants.** When any `bedrock_models[]` row has `mantle_selector: web_search`, +pass `--mantle-selector web_search` (join every `mantle_*` row's `mantle_selector` with commas; the other +selectors add nothing) and the helper adds a `BedrockWebSearch` statement: `bedrock-websearch:InvokeSearch` +and `bedrock-websearch:InvokeFetch` on resource `*` with an `aws:RequestedRegion` condition — the three +`bedrock-websearch` actions all take `*` and that global key is the service's only condition key +(, read 2026-11-09). The +generated client MUST set `"external_web_access": false` on the `web_search` tool unless the user +explicitly confirmed external web access in Clarify: the parameter defaults to `true`, and without the +`bedrock-websearch:ExternalWebAccess` action every Fetch then fails authorization before the cache is read +while the Responses request still returns 200 and may still carry `url_citation` annotations +(, "Controlling external web access", +read 2026-11-09). Only when the user confirmed external access, pass `--external-web-access` too and the +statement also grants `bedrock-websearch:ExternalWebAccess`; say in the report that request data may then +leave the AWS boundary. Web Search is served on the `bedrock-mantle` Responses API only, so the helper exits +1 with `reason: web_search_without_mantle_row` when no `--target` row is a bare Mantle id, and with +`reason: external_web_access_without_web_search` when the flag is passed without the selector — report +either verbatim. A plan with no `web_search` row gets no `bedrock-websearch:*` action; do not add one by hand. The script handles the dual-ARN pattern: foundation-model ARNs for plain model IDs and inference-profile ARNs for geo-prefixed IDs (e.g. `us.anthropic.claude-sonnet-4-6`). For any cross-region (CRIS) target it also emits a `BedrockGetInferenceProfile` statement (`bedrock:GetInferenceProfile` on the profile ARNs) — the documented prerequisite for invoking through a profile (, read 2026-11-07). For every runtime Responses / Chat Completions target — any proprietary GPT CRIS id by id form (GPT-5.6 Sol / Terra / Luna, the GPT-6 family, GPT-5.5 / GPT-5.4: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`, or their `inference-profile/` ARNs) whose own `--target` API is `responses` or `chat_completions` (legacy form: `--runtime-api` set to one of those, or omitted) — it adds a third resource, `arn:aws:bedrock:::project/default`, because a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, read 2026-11-07) and each of those model cards (read 2026-11-07) repeats the requirement; do not strip it from the report's policy excerpt. A row on `converse` or `messages` and every Anthropic / Nova / gpt-oss / embedding id get no project resource. The output is a ready-to-use IAM policy JSON file. @@ -924,7 +949,17 @@ line `*Monthly estimate unavailable — sample has no time-span information.*` retained op>). A 403 on the second call means the policy's `BedrockStoredResponseOps` statement (`bedrock:GetInvoke` <, `CancelInvoke`, `DeleteInvoke`> on `arn:aws:bedrock:::project/default`) is not attached — creation passing does not - prove retrieval is authorized. + prove retrieval is authorized. + - + **Tool-level permission check** (a plain model ping passing does not prove the Web Search grant): + under the attached identity run `client.responses.create(model="", + input="", tools=[{"type": "web_search", "external_web_access": False}])` + against the Mantle base URL, confirm the output contains a `web_search_call` item or a `url_citation` + annotation, and check CloudTrail Web Search data events for a denied `InvokeSearch` / `InvokeFetch` — + a denied call is not disclosed in the model response. 3. **Run tests:** diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md index f3f943622..9f54a00af 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -121,9 +121,32 @@ stored response authorizes those actions on the project (same page, "Permissions `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits that `BedrockStoredResponseOps` statement for a runtime `responses` row only and refuses (`reason: stored_response_ops_no_responses_row`) for a `converse`, `messages` or `chat_completions` plan, -which has no stored responses. `setup_bedrock.sh`'s verification then also retrieves the response it -just created (`r = client.responses.create(...)`, then `client.responses.retrieve(r.id)`): a 403 on -the second call means the grant is missing even though creation passed. +which has no stored responses on either endpoint. A bare Mantle id on `responses` (a +`mantle_openai_responses` row) satisfies retained ops through the `BedrockMantleInference` statement: +stored responses are retrievable on Mantle at `GET /v1/responses/{id}` under +`bedrock-mantle:CreateInference` (same page, "Permissions", read 2026-11-09), so no `bedrock:GetInvoke` +is added and the helper does not refuse. `setup_bedrock.sh`'s verification then also retrieves the +response it just created (`r = client.responses.create(...)`, then `client.responses.retrieve(r.id)`): +a 403 on the second call means the grant is missing even though creation passed. + +**Web Search grants.** When a `bedrock_models[]` row has `mantle_selector: web_search`, pass +`iam_policy.py --mantle-selector web_search` and the policy gains a `BedrockWebSearch` statement: +`bedrock-websearch:InvokeSearch` and `bedrock-websearch:InvokeFetch` on resource `*` with an +`aws:RequestedRegion` condition (all three `bedrock-websearch` actions take `*`; that is the service's +only condition key — , +read 2026-11-09). The generated client sets `"external_web_access": false` on the `web_search` tool +by default: the parameter defaults to `true`, and without `bedrock-websearch:ExternalWebAccess` every +Fetch then fails authorization before the cache is read while the request still returns 200 +(, "Controlling external web +access", read 2026-11-09). Only when the user explicitly confirmed external web access in Clarify, +pass `--external-web-access` as well (adds `bedrock-websearch:ExternalWebAccess`) and leave the +parameter `true`. Web Search is served on `bedrock-mantle` Responses only, so the helper refuses +(`reason: web_search_without_mantle_row`) when no row is a bare Mantle id; a plan without a +`web_search` row gets no `bedrock-websearch:*` action. Step 5's verification then also runs a +tool-level check: `client.responses.create(model=, input=, +tools=[{"type": "web_search", "external_web_access": False}])`, confirm a `web_search_call` item or +a `url_citation` annotation in the output, and check CloudTrail Web Search data events for a denied +`InvokeSearch` / `InvokeFetch` — a plain model ping passing does not prove the grant. ## Step 3B: `gateway_config.{yaml,py,json}` (gateway users) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index 11af0d657..b8a8b69c0 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -352,8 +352,19 @@ limited to: and the app's existing retrieve is denied. `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits the `BedrockStoredResponseOps` statement for a runtime `responses` row only and refuses for a `converse` / `messages` / `chat_completions` plan (`reason: stored_response_ops_no_responses_row`), which has no stored - responses. Verify with create-plus-retrieve: `r = client.responses.create(...)` then - `client.responses.retrieve(r.id)` under the attached role. + responses on either endpoint; a bare Mantle id on `responses` is covered by the `BedrockMantleInference` + statement (stored responses are retrievable on Mantle at `GET /v1/responses/{id}` under + `bedrock-mantle:CreateInference`, same page, read 2026-11-09) and gets no `bedrock:GetInvoke`. Verify with + create-plus-retrieve: `r = client.responses.create(...)` then `client.responses.retrieve(r.id)` under the + attached role. +6. Web Search → a row with `mantle_selector: web_search` needs `bedrock-websearch:InvokeSearch` and + `bedrock-websearch:InvokeFetch` on `*` (`iam_policy.py --mantle-selector web_search` emits them with an + `aws:RequestedRegion` condition); the client sets `"external_web_access": false` on the `web_search` tool + unless the user explicitly confirmed external access, in which case `--external-web-access` also grants + `bedrock-websearch:ExternalWebAccess` — the parameter defaults to `true` and every Fetch fails authorization + without that action (, + , read 2026-11-09). A plan + without a `web_search` row gets no `bedrock-websearch:*` action. **Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md index f97fb2f25..6049f6ede 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -216,12 +216,12 @@ Generate `ai-migration/setup_bedrock.sh`. - Dry-run by default (`--execute` flag to run for real) - Step 1 — Request model access: List each model from `aws-design-ai.json` → `bedrock_models[].aws_model_id` and the embedding model -- Step 2 — Create IAM role: Trust policy for the compute platform (Lambda, ECS, or EC2 based on `aws-design.json` if present). Match inference permissions to the selected endpoint. Mantle needs `bedrock-mantle:CreateInference` scoped to `arn:aws:bedrock-mantle:{region}:{account_id}:project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` when using bearer credentials. Runtime needs `bedrock:InvokeModel` / `bedrock:InvokeModelWithResponseStream` against the selected foundation-model ARN and, for CRIS, the exact inference-profile ARN. For EVERY `runtime_openai_cris` row whose `required_client_api` is `responses` or `chat_completions` — classified by id form, not model family: any `us.`/`global.` (and, for GPT-5.6 Terra and Luna, `in.`) `openai.gpt-*` CRIS id, so GPT-5.6 Sol / Terra / Luna, the GPT-6 family, GPT-5.5 and GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default`: a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, "Permissions", read 2026-11-07), and each of those model cards (read 2026-11-07) repeats the requirement for its runtime CRIS ids. `iam_policy.py --target =` (one pair per `bedrock_models[]` row, each row's own API, so a GPT Chat / Responses secondary beside a Claude Messages primary still gets it) emits that third ARN for exactly those rows. A row with `required_client_api: "converse"` (the Converse-only application-inference-profile route) and every Anthropic / Nova / gpt-oss id get no such grant; a bare `openai.gpt-5.5` / `openai.gpt-5.4` / `openai.gpt-5.6-*` / `openai.gpt-6-*` id is a mantle target and gets no such grant. Runtime actions do not authorize Mantle. **Retained stored-response operations:** when the source code keeps `client.responses.retrieve(id)` (or `.cancel(id)` / `.delete(id)`; scan for `responses.retrieve(` / `responses.cancel(` / `responses.delete(`), a runtime `responses` row ALSO needs `bedrock:GetInvoke` (and `bedrock:CancelInvoke` / `bedrock:DeleteInvoke` only for the ops actually retained) on the same `arn:aws:bedrock:{region}:{account}:project/default` — creating a response authorizes `bedrock:InvokeModel` on the target and the project, while retrieving, canceling or deleting a stored response authorizes those actions on the project (same page, "Permissions", read 2026-11-08); without them creation and the ping pass and the app's existing retrieve is denied. `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits that `BedrockStoredResponseOps` statement for a runtime `responses` row only and refuses (`reason: stored_response_ops_no_responses_row`) for a `converse`, `messages` or `chat_completions` plan, which has no stored responses. Step 5's verification must then also retrieve the response it just created. +- Step 2 — Create IAM role: Trust policy for the compute platform (Lambda, ECS, or EC2 based on `aws-design.json` if present). Match inference permissions to the selected endpoint. Mantle needs `bedrock-mantle:CreateInference` scoped to `arn:aws:bedrock-mantle:{region}:{account_id}:project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` when using bearer credentials. Runtime needs `bedrock:InvokeModel` / `bedrock:InvokeModelWithResponseStream` against the selected foundation-model ARN and, for CRIS, the exact inference-profile ARN. For EVERY `runtime_openai_cris` row whose `required_client_api` is `responses` or `chat_completions` — classified by id form, not model family: any `us.`/`global.` (and, for GPT-5.6 Terra and Luna, `in.`) `openai.gpt-*` CRIS id, so GPT-5.6 Sol / Terra / Luna, the GPT-6 family, GPT-5.5 and GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default`: a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, "Permissions", read 2026-11-07), and each of those model cards (read 2026-11-07) repeats the requirement for its runtime CRIS ids. `iam_policy.py --target =` (one pair per `bedrock_models[]` row, each row's own API, so a GPT Chat / Responses secondary beside a Claude Messages primary still gets it) emits that third ARN for exactly those rows. A row with `required_client_api: "converse"` (the Converse-only application-inference-profile route) and every Anthropic / Nova / gpt-oss id get no such grant; a bare `openai.gpt-5.5` / `openai.gpt-5.4` / `openai.gpt-5.6-*` / `openai.gpt-6-*` id is a mantle target and gets no such grant. Runtime actions do not authorize Mantle. **Retained stored-response operations:** when the source code keeps `client.responses.retrieve(id)` (or `.cancel(id)` / `.delete(id)`; scan for `responses.retrieve(` / `responses.cancel(` / `responses.delete(`), a runtime `responses` row ALSO needs `bedrock:GetInvoke` (and `bedrock:CancelInvoke` / `bedrock:DeleteInvoke` only for the ops actually retained) on the same `arn:aws:bedrock:{region}:{account}:project/default` — creating a response authorizes `bedrock:InvokeModel` on the target and the project, while retrieving, canceling or deleting a stored response authorizes those actions on the project (same page, "Permissions", read 2026-11-08); without them creation and the ping pass and the app's existing retrieve is denied. `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits that `BedrockStoredResponseOps` statement for a runtime `responses` row only and refuses (`reason: stored_response_ops_no_responses_row`) for a `converse`, `messages` or `chat_completions` plan, which has no stored responses on either endpoint. A bare Mantle id on `responses` (a `mantle_openai_responses` row) satisfies retained ops through the `BedrockMantleInference` statement: stored responses are retrievable on Mantle at `GET /v1/responses/{id}` under `bedrock-mantle:CreateInference` (same page, "Permissions", read 2026-11-09), so no `bedrock:GetInvoke` is added and the helper does not refuse. Step 5's verification must then also retrieve the response it just created. **Web Search grants:** when a `bedrock_models[]` row has `mantle_selector: web_search`, pass `iam_policy.py --mantle-selector web_search` and the policy gains a `BedrockWebSearch` statement — `bedrock-websearch:InvokeSearch` and `bedrock-websearch:InvokeFetch` on resource `*` with an `aws:RequestedRegion` condition (all three `bedrock-websearch` actions take `*`; that is the service's only condition key — , read 2026-11-09). The generated client sets `"external_web_access": false` on the `web_search` tool by default: the parameter defaults to `true`, and without `bedrock-websearch:ExternalWebAccess` every Fetch then fails authorization before the cache is read while the request still returns 200 (, "Controlling external web access", read 2026-11-09). Only when the user explicitly confirmed external web access in Clarify, pass `--external-web-access` as well (adds `bedrock-websearch:ExternalWebAccess`) and leave the parameter `true`. Web Search is served on `bedrock-mantle` Responses only, so the helper refuses (`reason: web_search_without_mantle_row`) when no row is a bare Mantle id; a plan without a `web_search` row gets no `bedrock-websearch:*` action. - If the runtime client uses `BEDROCK_INFERENCE_PROFILE_ARN` from the generated monitoring output, also authorize that exact `application-inference-profile/…` ARN, retaining the backing model/CRIS resources needed for the selected model and fallback. Resolve it from the same output used by the client; do not grant every application profile. Re-run setup after enabling or changing this override so the effective invocation target and IAM remain aligned. See [inference-profile IAM prerequisites](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-prereq.html). - Compliance carry-through: if `preferences.json` → `design_constraints.compliance` is set and not `none`/`unknown` (full-flow Q2 / AI-only Q1.5), the script and its printed notes MUST reflect the design's Step 0.7 constraints — `AWS_REGION` from the compliance-constrained region (GovCloud for fedramp, EU for gdpr), `eu.` inference-profile model IDs for gdpr, and for hipaa a printed warning that Bedrock invocation logs retain original content (KMS-encrypt + restrict IAM on the log group before enabling) - Step 3 — Print required environment variables: `AWS_REGION`, `AI_PROVIDER=bedrock`, model IDs - Step 4 — Check quota: Query current TPM quota for the primary model via `aws service-quotas get-service-quota`. If `aws-design-ai.json` → `ai_architecture.quota_risk` is `"high"` or `"medium"`, print warning: "⚠️ Your token volume may exceed default Bedrock quotas. Request a quota increase via Service Quotas console (allow 1–5 business days)." Include the `aws service-quotas request-service-quota-increase` command template. -- Step 5 — Verification: Run `test_comparison.py --target-only --quick` using the same selected API, model ID and region as the generated client. A Mantle selection must never run a Converse probe; a runtime selection must preserve its validated profile ID. When Step 2 granted stored-response operations, add a create-plus-retrieve check on the runtime Responses API: `r = client.responses.create(model=, input="ping")` then `client.responses.retrieve(r.id)` (and `.cancel` / `.delete` for each other retained op) — a 403 on the second call means the `BedrockStoredResponseOps` grant is missing even though creation passed. +- Step 5 — Verification: Run `test_comparison.py --target-only --quick` using the same selected API, model ID and region as the generated client. A Mantle selection must never run a Converse probe; a runtime selection must preserve its validated profile ID. When Step 2 granted stored-response operations, add a create-plus-retrieve check on the runtime Responses API: `r = client.responses.create(model=, input="ping")` then `client.responses.retrieve(r.id)` (and `.cancel` / `.delete` for each other retained op) — a 403 on the second call means the `BedrockStoredResponseOps` grant is missing even though creation passed. When Step 2 granted Web Search, add a tool-level check against the Mantle base URL: `client.responses.create(model=, input=, tools=[{"type": "web_search", "external_web_access": False}])`, confirm a `web_search_call` item or a `url_citation` annotation in the output, and check CloudTrail Web Search data events for a denied `InvokeSearch` / `InvokeFetch` — a plain model ping passing does not prove the grant. - If `$MIGRATION_DIR/terraform/` exists, print coordination note: "Ensure the IAM role is referenced in compute.tf task definitions" - Use region from `preferences.json` → `design_constraints.target_region` diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index 11af0d657..b8a8b69c0 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -352,8 +352,19 @@ limited to: and the app's existing retrieve is denied. `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits the `BedrockStoredResponseOps` statement for a runtime `responses` row only and refuses for a `converse` / `messages` / `chat_completions` plan (`reason: stored_response_ops_no_responses_row`), which has no stored - responses. Verify with create-plus-retrieve: `r = client.responses.create(...)` then - `client.responses.retrieve(r.id)` under the attached role. + responses on either endpoint; a bare Mantle id on `responses` is covered by the `BedrockMantleInference` + statement (stored responses are retrievable on Mantle at `GET /v1/responses/{id}` under + `bedrock-mantle:CreateInference`, same page, read 2026-11-09) and gets no `bedrock:GetInvoke`. Verify with + create-plus-retrieve: `r = client.responses.create(...)` then `client.responses.retrieve(r.id)` under the + attached role. +6. Web Search → a row with `mantle_selector: web_search` needs `bedrock-websearch:InvokeSearch` and + `bedrock-websearch:InvokeFetch` on `*` (`iam_policy.py --mantle-selector web_search` emits them with an + `aws:RequestedRegion` condition); the client sets `"external_web_access": false` on the `web_search` tool + unless the user explicitly confirmed external access, in which case `--external-web-access` also grants + `bedrock-websearch:ExternalWebAccess` — the parameter defaults to `true` and every Fetch fails authorization + without that action (, + , read 2026-11-09). A plan + without a `web_search` row gets no `bedrock-websearch:*` action. **Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py index 3418a820a..ebbf6f8aa 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/iam_policy.py @@ -19,18 +19,50 @@ TARGET_APIS = RUNTIME_APIS | {"invoke"} # Stored-response operations a source app may RETAIN after the rewrite, and the action -# each one authorizes on the default project. Creating a response authorizes -# `bedrock:InvokeModel` on the inference target and the project; retrieving, canceling -# and deleting a stored response authorize these three, each on the project -# (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, -# "Permissions", read 2026-11-08). They exist only on the runtime Responses API: -# Converse, Anthropic Messages and Chat Completions rows never get them. +# each one authorizes on the default project for a bedrock-RUNTIME Responses row. +# Creating a response on bedrock-runtime authorizes `bedrock:InvokeModel` on the +# inference target and the project; retrieving, canceling and deleting a stored +# response authorize these three, each on the project. Stored responses exist on the +# Responses API of EITHER endpoint: on bedrock-mantle a stored response is retrieved +# at `GET /v1/responses/{id}` and inference authorizes `bedrock-mantle:CreateInference`, +# so a Mantle Responses row is covered by the Mantle statement and gets none of these +# runtime actions (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, +# "Prerequisites → Permissions" and "Using the Responses API on the bedrock-runtime +# endpoint → Permissions", read 2026-11-09). Converse, Anthropic Messages and Chat +# Completions rows never carry stored responses on any endpoint. STORED_RESPONSE_OPS = { "retrieve": "bedrock:GetInvoke", "cancel": "bedrock:CancelInvoke", "delete": "bedrock:DeleteInvoke", } +# Mantle selectors a plan row may carry (bedrock_models[].mantle_selector); the +# vocabulary of skills/shared/ai/bedrock-endpoint-policy.md. Only `web_search` adds a +# grant — the others are satisfied by the Mantle statements and add nothing here. +MANTLE_SELECTORS = frozenset({ + "background_inference", + "server_side_tools", + "web_search", + "projects", + "workspaces", + "mantle_only_model_or_api", + "in_region_processing", +}) + +# Web Search on Amazon Bedrock is served on the bedrock-mantle Responses API only +# (GPT-5.4 and later). Its IAM prefix is `bedrock-websearch`; InvokeSearch, +# InvokeFetch and ExternalWebAccess are all ACTIONS on resource `*` and +# `aws:RequestedRegion` is the only condition key. The request parameter +# `external_web_access` defaults to `true`; without the ExternalWebAccess action every +# Fetch then fails authorization before the cache is read while the request still +# returns 200, so the generated client sets `"external_web_access": false` unless the +# user confirmed external access and `--external-web-access` adds the third action +# (https://docs.aws.amazon.com/bedrock/latest/userguide/web-search.html and +# https://docs.aws.amazon.com/bedrock/latest/userguide/security-web-search.html, +# read 2026-11-09). +WEB_SEARCH_ACTIONS = ("bedrock-websearch:InvokeSearch", "bedrock-websearch:InvokeFetch") +WEB_SEARCH_EXTERNAL_ACTION = "bedrock-websearch:ExternalWebAccess" + # Execution lanes this helper refuses to authorize. `mantle_messages` (Anthropic SDK # on the bedrock-mantle `/anthropic` endpoint, selected by the `workspaces` mantle # selector) is deferred: llm-to-bedrock has no preflight, evaluation or rewrite route @@ -236,6 +268,29 @@ def parse_stored_response_ops(specs: list[str]) -> list[str]: return ops +def parse_mantle_selectors(specs: list[str]) -> list[str]: + """Pure: parse repeatable `--mantle-selector SEL[,SEL]` specs into an ordered, + de-duplicated list of MANTLE_SELECTORS values. Fails CLOSED on any other value so + a grant is never added on a misspelt selector.""" + selectors: list[str] = [] + bad: list[str] = [] + for spec in specs: + for item in spec.split(","): + item = item.strip() + if not item: + continue + if item not in MANTLE_SELECTORS: + bad.append(item) + elif item not in selectors: + selectors.append(item) + if bad: + raise ValueError( + "Every --mantle-selector value must be one of " + f"{' | '.join(sorted(MANTLE_SELECTORS))} (bedrock_models[].mantle_selector of a " + f"mantle_* row). Unknown: {', '.join(bad)}.") + return selectors + + def validated_target_pairs(rows: list[dict], validated_mapping: list[str] | dict[str, str], primary_model_id: str | None = None, primary_api: str | None = None) -> list[tuple[str, str]]: @@ -287,7 +342,9 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, runtime_api: str | None = None, migration_path: str | None = None, target_apis: dict[str, str | set[str]] | None = None, - stored_response_ops: list[str] | None = None) -> dict: + stored_response_ops: list[str] | None = None, + mantle_selectors: list[str] | None = None, + external_web_access: bool = False) -> dict: """Build a scoped IAM policy covering exactly the given model IDs. `runtime_api` is the plan's selected API (`responses`, `chat_completions`, @@ -304,15 +361,27 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, runtime `bedrock:InvokeModel*` for a bare Claude id that the lane would never send to bedrock-runtime. `stored_response_ops` lists the stored-response operations the source app - RETAINS (`retrieve` / `cancel` / `delete`, see STORED_RESPONSE_OPS). Each adds - its action — `bedrock:GetInvoke` / `CancelInvoke` / `DeleteInvoke` — on the - default project, which is what retrieving, canceling or deleting a stored - response authorizes (inference-responses-api.html, "Permissions", read - 2026-11-08). Only a runtime GPT CRIS row on `responses` can carry stored + RETAINS (`retrieve` / `cancel` / `delete`, see STORED_RESPONSE_OPS). The grant + is dispatched by the endpoint of each `responses` row: a runtime GPT CRIS row + adds its action — `bedrock:GetInvoke` / `CancelInvoke` / `DeleteInvoke` — on + the default project, which is what retrieving, canceling or deleting a stored + response authorizes on bedrock-runtime; a bare Mantle id on `responses` is + already covered by the `BedrockMantleInference` statement (Mantle inference, + stored-response retrieval included, authorizes `bedrock-mantle:CreateInference` + — inference-responses-api.html, "Permissions", read 2026-11-09) and adds no + runtime action. Only a `responses` row on either endpoint can carry stored responses, so declaring ops when no such row exists raises `ValueError`: a Converse, Anthropic Messages or Chat Completions plan never gets these grants. - - Emits up to five statements, depending on which endpoints the targets use: + `mantle_selectors` lists the `bedrock_models[].mantle_selector` values of the + plan's `mantle_*` rows (see MANTLE_SELECTORS; unknown → `ValueError`). Only + `web_search` adds a statement: `bedrock-websearch:InvokeSearch` + `InvokeFetch` + on `*`, conditioned on `aws:RequestedRegion`, and — only when + `external_web_access` is True because the user explicitly confirmed external + web access — `bedrock-websearch:ExternalWebAccess`. Web Search is served on + the bedrock-mantle Responses API only, so `web_search` without a Mantle row, + or `external_web_access` without `web_search`, raises `ValueError`. + + Emits up to six statements, depending on which endpoints the targets use: - `bedrock:InvokeModel*` scoped to foundation-model ARNs (plain IDs) and inference-profile ARNs (geo-prefixed IDs), for bedrock-runtime targets - `bedrock:GetInferenceProfile` scoped to the inference-profile ARNs, when any @@ -325,6 +394,8 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, - `bedrock-mantle:CreateInference` / `Get*` / `List*` scoped to the account's mantle projects, for mantle-only targets (SigV4 auth) - `bedrock-mantle:CallWithBearerToken` on `*`, for Bedrock API-key auth + - `bedrock-websearch:InvokeSearch` / `InvokeFetch` (+ `ExternalWebAccess` when + confirmed) on `*` with the region condition, for a `web_search` Mantle row A statement is omitted entirely when no target needs it — notably, an all-mantle run must not emit an InvokeModel statement with an empty Resource @@ -351,8 +422,23 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, if op not in STORED_RESPONSE_OPS: raise ValueError(f"Unknown stored-response op {op!r}; expected one of " f"{sorted(STORED_RESPONSE_OPS)}") + mantle_selectors = list(mantle_selectors or []) + for selector in mantle_selectors: + if selector not in MANTLE_SELECTORS: + raise ValueError(f"Unknown mantle selector {selector!r}; expected one of " + f"{sorted(MANTLE_SELECTORS)}") runtime_ids = [m for m in model_ids if not is_mantle_model(m, region)] mantle_ids = [m for m in model_ids if is_mantle_model(m, region)] + web_search = "web_search" in mantle_selectors + if web_search and not mantle_ids: + raise ValueError( + "web_search selector declared but no --target row is a bare Mantle id; Web Search is " + "served on the bedrock-mantle Responses API only (web-search.html, read 2026-11-09), so " + "a bedrock-runtime plan never gets bedrock-websearch:InvokeSearch / InvokeFetch") + if external_web_access and not web_search: + raise ValueError( + "external web access confirmed but no web_search selector was declared; " + "bedrock-websearch:ExternalWebAccess is granted only alongside a Web Search row") statements = [] @@ -401,27 +487,36 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, }) if stored_response_ops: - # Stored responses exist only on the runtime Responses API, so the grant is - # keyed on a row that is BOTH a runtime GPT CRIS target AND on `responses` - # (the same rows that get the default-project InvokeModel grant for - # creation). A Converse / Messages / Chat Completions plan, or a legacy - # id-only call with no API, fails closed instead of receiving GetInvoke. - responses_rows = [ + # Stored responses exist on the Responses API of either endpoint, so the grant + # is dispatched by the endpoint of each `responses` row. A runtime GPT CRIS + # row on `responses` (the same rows that get the default-project InvokeModel + # grant for creation) gets GetInvoke / CancelInvoke / DeleteInvoke on the + # project. A bare Mantle id on `responses` retrieves its stored responses at + # `GET /v1/responses/{id}` under `bedrock-mantle:CreateInference`, which the + # BedrockMantleInference statement below already grants — no runtime action is + # added for it. A Converse / Messages / Chat Completions plan, or a legacy + # id-only call with no API, has no Responses row on any endpoint and fails + # closed instead of receiving GetInvoke. + def _apis(mid: str) -> set: + return target_apis.get(mid) or {runtime_api} + runtime_responses_rows = [ mid for mid in runtime_ids - if "responses" in (target_apis.get(mid) or {runtime_api}) - and requires_default_project_grant(mid, "responses")] - if not responses_rows: + if "responses" in _apis(mid) and requires_default_project_grant(mid, "responses")] + mantle_responses_rows = [mid for mid in mantle_ids if "responses" in _apis(mid)] + if not runtime_responses_rows and not mantle_responses_rows: raise ValueError( f"stored-response ops {stored_response_ops} declared but no --target row is a " - "runtime GPT CRIS id on the `responses` API; retrieving / canceling / deleting a " - "stored response exists only there, so a converse / messages / chat_completions " - "plan never gets bedrock:GetInvoke / CancelInvoke / DeleteInvoke") - statements.append({ - "Sid": "BedrockStoredResponseOps", - "Effect": "Allow", - "Action": sorted(STORED_RESPONSE_OPS[op] for op in stored_response_ops), - "Resource": [default_project_arn(region, account_id)], - }) + "runtime GPT CRIS id or a bare Mantle id on the `responses` API; retrieving / " + "canceling / deleting a stored response exists only there, so a converse / " + "messages / chat_completions plan never gets bedrock:GetInvoke / CancelInvoke / " + "DeleteInvoke") + if runtime_responses_rows: + statements.append({ + "Sid": "BedrockStoredResponseOps", + "Effect": "Allow", + "Action": sorted(STORED_RESPONSE_OPS[op] for op in stored_response_ops), + "Resource": [default_project_arn(region, account_id)], + }) if mantle_ids: statements.append({ @@ -445,6 +540,23 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, "Resource": "*", }) + if web_search: + # All three bedrock-websearch actions take resource `*`; the service's own + # example scopes them with `aws:RequestedRegion`, its only condition key, and + # Web Search is strictly regional (security-web-search.html, web-search.html, + # read 2026-11-09). ExternalWebAccess is added only on explicit confirmation; + # otherwise the generated client must send `"external_web_access": false`. + actions = list(WEB_SEARCH_ACTIONS) + if external_web_access: + actions.append(WEB_SEARCH_EXTERNAL_ACTION) + statements.append({ + "Sid": "BedrockWebSearch", + "Effect": "Allow", + "Action": sorted(actions), + "Resource": "*", + "Condition": {"StringEquals": {"aws:RequestedRegion": region}}, + }) + return {"Version": "2012-10-17", "Statement": statements} @@ -475,8 +587,25 @@ def generate_policy(model_ids: list[str], region: str, account_id: str, help="retrieve | cancel | delete (comma-joined or repeated): the " "client.responses.(id) calls the source app RETAINS. Adds " "bedrock:GetInvoke / CancelInvoke / DeleteInvoke on project/default " - "for a runtime GPT CRIS row on `responses`; exits 1 with reason " - "stored_response_ops_no_responses_row when no such row exists.") + "for a runtime GPT CRIS row on `responses`; a bare Mantle id on " + "`responses` is covered by the Mantle statement and adds nothing; " + "exits 1 with reason stored_response_ops_no_responses_row when no " + "`responses` row exists on either endpoint.") + parser.add_argument("--mantle-selector", action="append", default=[], + help="bedrock_models[].mantle_selector of every mantle_* row (comma-joined " + "or repeated). `web_search` adds bedrock-websearch:InvokeSearch + " + "InvokeFetch on * with an aws:RequestedRegion condition; the other " + "selectors add nothing. Exits 1 with reason unknown_mantle_selector " + "for an unknown value, or web_search_without_mantle_row when no " + "--target row is a bare Mantle id.") + parser.add_argument("--external-web-access", action="store_true", + help="Pass ONLY when the user explicitly confirmed external web access: " + "adds bedrock-websearch:ExternalWebAccess to the web_search " + "statement. Without it the generated client must send " + "\"external_web_access\": false (the parameter defaults to true and " + "every Fetch then fails authorization). Exits 1 with reason " + "external_web_access_without_web_search when no web_search selector " + "is declared.") args = parser.parse_args() def _fail(reason: str, detail: str) -> None: @@ -504,14 +633,28 @@ def _fail(reason: str, detail: str) -> None: stored_response_ops = parse_stored_response_ops(args.stored_response_ops) except ValueError as e: _fail("unknown_stored_response_op", str(e)) + try: + mantle_selectors = parse_mantle_selectors(args.mantle_selector) + except ValueError as e: + _fail("unknown_mantle_selector", str(e)) try: policy = generate_policy(model_ids, args.region, args.account_id, runtime_api=args.runtime_api if not args.target else None, migration_path=args.migration_path, target_apis=target_apis, - stored_response_ops=stored_response_ops) + stored_response_ops=stored_response_ops, + mantle_selectors=mantle_selectors, + external_web_access=args.external_web_access) except ValueError as e: - _fail("stored_response_ops_no_responses_row" if "stored-response ops" in str(e) - else "unknown_runtime_api", str(e)) + message = str(e) + if message.startswith("stored-response ops"): + reason = "stored_response_ops_no_responses_row" + elif message.startswith("web_search selector"): + reason = "web_search_without_mantle_row" + elif message.startswith("external web access"): + reason = "external_web_access_without_web_search" + else: + reason = "unknown_runtime_api" + _fail(reason, message) output = json.dumps(policy, indent=2) + "\n" if args.output: diff --git a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy_round2.py b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy_round2.py index af7a203c0..ed1ffcd72 100644 --- a/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy_round2.py +++ b/plugins/aws-startup-advisor/skills/llm-to-bedrock/scripts/test_iam_policy_round2.py @@ -23,9 +23,12 @@ from iam_policy import ( # noqa: E402 STORED_RESPONSE_OPS, + WEB_SEARCH_ACTIONS, + WEB_SEARCH_EXTERNAL_ACTION, default_project_arn, generate_policy, inference_profile_arn, + parse_mantle_selectors, parse_stored_response_ops, parse_targets, target_apis_by_id, @@ -36,6 +39,7 @@ ACCOUNT = "123456789012" PROJECT = default_project_arn(REGION, ACCOUNT) SOL, TERRA = "us.openai.gpt-5.6-sol", "us.openai.gpt-5.6-terra" +MANTLE_SOL, MANTLE_TERRA = "openai.gpt-6-sol", "openai.gpt-5.6-terra" CLAUDE = "global.anthropic.claude-sonnet-5" EMBED = "amazon.titan-embed-text-v2:0" @@ -134,7 +138,7 @@ def test_no_ops_means_no_stored_response_statement(self): ({CLAUDE: "messages", EMBED: "invoke"}, [CLAUDE, EMBED]), ]) def test_converse_messages_chat_and_embedding_rows_never_get_the_grant(self, target_apis, ids): - with pytest.raises(ValueError, match="stored-response ops .* no --target row is a runtime GPT CRIS id on the `responses` API"): + with pytest.raises(ValueError, match="stored-response ops .* no --target row is a runtime GPT CRIS id or a bare Mantle id on the `responses` API"): generate_policy(ids, REGION, ACCOUNT, target_apis=target_apis, stored_response_ops=["retrieve"]) def test_mixed_plan_grants_once_for_the_responses_row_beside_a_messages_primary(self): @@ -144,9 +148,40 @@ def test_mixed_plan_grants_once_for_the_responses_row_beside_a_messages_primary( "Sid": "BedrockStoredResponseOps", "Effect": "Allow", "Action": ["bedrock:GetInvoke"], "Resource": [PROJECT]} - def test_bare_mantle_id_on_responses_never_gets_runtime_get_invoke(self): - with pytest.raises(ValueError, match="stored-response ops"): - generate_policy(["openai.gpt-5.6-sol"], REGION, ACCOUNT, target_apis={"openai.gpt-5.6-sol": "responses"}, + # PR 588 F12: the grant is dispatched by endpoint. A Mantle Responses row retrieves its + # stored responses at GET /v1/responses/{id} under bedrock-mantle:CreateInference + # (inference-responses-api.html, "Permissions", read 2026-11-09), so the Mantle policy IS + # the stored-response policy — no runtime GetInvoke, and no fail-closed error. + def test_all_mantle_responses_plan_with_retained_retrieve_keeps_the_mantle_policy(self): + pol = generate_policy([MANTLE_SOL], REGION, ACCOUNT, target_apis={MANTLE_SOL: "responses"}, + stored_response_ops=["retrieve"]) + assert _sids(pol) == ["BedrockMantleInference", "BedrockMantleCallWithBearerToken"] + assert "bedrock-mantle:CreateInference" in _stmt(pol, "BedrockMantleInference")["Action"] + assert not any("Invoke" in a and a.startswith("bedrock:") for s in pol["Statement"] for a in s["Action"]) + legacy = generate_policy([MANTLE_SOL], REGION, ACCOUNT, runtime_api="responses", + stored_response_ops=["retrieve", "cancel", "delete"]) + assert _sids(legacy) == ["BedrockMantleInference", "BedrockMantleCallWithBearerToken"] + + def test_runtime_responses_row_still_gets_get_invoke_on_project_default(self): + pol = generate_policy([SOL], REGION, ACCOUNT, target_apis={SOL: "responses"}, + stored_response_ops=["retrieve"]) + assert _stmt(pol, "BedrockStoredResponseOps") == { + "Sid": "BedrockStoredResponseOps", "Effect": "Allow", + "Action": ["bedrock:GetInvoke"], "Resource": [PROJECT]} + assert "BedrockMantleInference" not in _sids(pol) + + def test_mixed_runtime_and_mantle_responses_rows_grant_get_invoke_once_and_keep_mantle(self): + pol = generate_policy([SOL, MANTLE_SOL], REGION, ACCOUNT, + target_apis={SOL: "responses", MANTLE_SOL: "responses"}, + stored_response_ops=["retrieve"]) + assert _sids(pol) == ["BedrockInvokeModelScoped", "BedrockGetInferenceProfile", "BedrockStoredResponseOps", + "BedrockMantleInference", "BedrockMantleCallWithBearerToken"] + assert _stmt(pol, "BedrockStoredResponseOps")["Resource"] == [PROJECT] + assert sum(1 for s in pol["Statement"] if "bedrock:GetInvoke" in s["Action"]) == 1 + + def test_mantle_chat_only_plan_with_retained_ops_still_fails_closed(self): + with pytest.raises(ValueError, match="stored-response ops .* no --target row is a runtime GPT CRIS id or a bare Mantle id"): + generate_policy(["openai.gpt-5.5"], REGION, ACCOUNT, target_apis={"openai.gpt-5.5": "chat_completions"}, stored_response_ops=["retrieve"]) def test_legacy_models_form_needs_an_explicit_responses_api(self): @@ -188,3 +223,94 @@ def test_cli_refuses_an_unknown_op(self): result = self._cli("--target", f"{SOL}=responses", "--stored-response-ops", "list") assert result.returncode == 1 assert json.loads(result.stdout)["reason"] == "unknown_stored_response_op" + + def test_cli_all_mantle_responses_plan_with_retained_retrieve_exits_0_with_the_mantle_policy(self): + result = self._cli("--target", f"{MANTLE_SOL}=responses", "--stored-response-ops", "retrieve") + assert result.returncode == 0, result.stdout + pol = json.loads(result.stdout) + assert _sids(pol) == ["BedrockMantleInference", "BedrockMantleCallWithBearerToken"] + assert "GetInvoke" not in result.stdout + + +# --- PR 588 F15: a confirmed web_search requirement is carried into IAM --- +# https://docs.aws.amazon.com/bedrock/latest/userguide/web-search.html and +# https://docs.aws.amazon.com/bedrock/latest/userguide/security-web-search.html (read 2026-11-09): +# actions bedrock-websearch:InvokeSearch / InvokeFetch / ExternalWebAccess, all on resource `*`, +# aws:RequestedRegion the only condition key; Web Search is served on bedrock-mantle Responses only. + +class TestWebSearchGrants: + def test_web_search_row_gets_search_and_fetch_on_star_with_the_region_condition(self): + pol = generate_policy([MANTLE_TERRA], REGION, ACCOUNT, target_apis={MANTLE_TERRA: "responses"}, + mantle_selectors=["web_search"]) + assert _sids(pol) == ["BedrockMantleInference", "BedrockMantleCallWithBearerToken", "BedrockWebSearch"] + assert _stmt(pol, "BedrockWebSearch") == { + "Sid": "BedrockWebSearch", "Effect": "Allow", + "Action": ["bedrock-websearch:InvokeFetch", "bedrock-websearch:InvokeSearch"], + "Resource": "*", + "Condition": {"StringEquals": {"aws:RequestedRegion": REGION}}} + assert WEB_SEARCH_EXTERNAL_ACTION not in result_actions(pol) + + def test_confirmed_external_web_access_adds_the_third_action(self): + pol = generate_policy([MANTLE_TERRA], REGION, ACCOUNT, target_apis={MANTLE_TERRA: "responses"}, + mantle_selectors=["web_search"], external_web_access=True) + assert _stmt(pol, "BedrockWebSearch")["Action"] == [ + "bedrock-websearch:ExternalWebAccess", "bedrock-websearch:InvokeFetch", "bedrock-websearch:InvokeSearch"] + assert set(WEB_SEARCH_ACTIONS) | {WEB_SEARCH_EXTERNAL_ACTION} == set(_stmt(pol, "BedrockWebSearch")["Action"]) + + @pytest.mark.parametrize("selectors", [None, [], ["in_region_processing"], ["projects"], + ["background_inference", "server_side_tools"]]) + def test_non_web_search_plans_get_no_bedrock_websearch_action(self, selectors): + pol = generate_policy([MANTLE_TERRA], REGION, ACCOUNT, target_apis={MANTLE_TERRA: "responses"}, + mantle_selectors=selectors) + assert "BedrockWebSearch" not in _sids(pol) + assert not any(a.startswith("bedrock-websearch:") for a in result_actions(pol)) + + def test_runtime_only_plan_with_web_search_fails_closed(self): + with pytest.raises(ValueError, match="web_search selector declared but no --target row is a bare Mantle id"): + generate_policy([SOL], REGION, ACCOUNT, target_apis={SOL: "responses"}, mantle_selectors=["web_search"]) + + def test_external_web_access_without_web_search_fails_closed(self): + with pytest.raises(ValueError, match="external web access confirmed but no web_search selector"): + generate_policy([MANTLE_TERRA], REGION, ACCOUNT, target_apis={MANTLE_TERRA: "responses"}, + external_web_access=True) + + def test_unknown_selector_fails_closed(self): + with pytest.raises(ValueError, match="Unknown mantle selector 'web-search'"): + generate_policy([MANTLE_TERRA], REGION, ACCOUNT, target_apis={MANTLE_TERRA: "responses"}, + mantle_selectors=["web-search"]) + with pytest.raises(ValueError, match="Every --mantle-selector value must be"): + parse_mantle_selectors(["web_search,web-search"]) + assert parse_mantle_selectors(["web_search,projects", " web_search "]) == ["web_search", "projects"] + + def _cli(self, *extra): + return subprocess.run( # nosec B603 + [sys.executable, str(Path(__file__).parent / "iam_policy.py"), + "--region", REGION, "--account-id", ACCOUNT, *extra], + capture_output=True, text=True) + + def test_cli_web_search_selector_emits_the_statement_for_a_mantle_row(self): + result = self._cli("--target", f"{MANTLE_TERRA}=responses", "--mantle-selector", "web_search") + assert result.returncode == 0, result.stdout + stmt = _stmt(json.loads(result.stdout), "BedrockWebSearch") + assert stmt["Action"] == ["bedrock-websearch:InvokeFetch", "bedrock-websearch:InvokeSearch"] + assert stmt["Condition"] == {"StringEquals": {"aws:RequestedRegion": REGION}} + with_external = self._cli("--target", f"{MANTLE_TERRA}=responses", "--mantle-selector", "web_search", + "--external-web-access") + assert WEB_SEARCH_EXTERNAL_ACTION in _stmt(json.loads(with_external.stdout), "BedrockWebSearch")["Action"] + + def test_cli_refuses_web_search_for_a_runtime_plan(self): + result = self._cli("--target", f"{SOL}=responses", "--mantle-selector", "web_search") + assert result.returncode == 1 + assert json.loads(result.stdout)["reason"] == "web_search_without_mantle_row" + assert "Statement" not in result.stdout + + def test_cli_refuses_an_unknown_selector_and_external_access_without_web_search(self): + unknown = self._cli("--target", f"{MANTLE_TERRA}=responses", "--mantle-selector", "web-search") + assert unknown.returncode == 1 and json.loads(unknown.stdout)["reason"] == "unknown_mantle_selector" + external = self._cli("--target", f"{MANTLE_TERRA}=responses", "--external-web-access") + assert external.returncode == 1 + assert json.loads(external.stdout)["reason"] == "external_web_access_without_web_search" + + +def result_actions(pol): + return [a for s in pol["Statement"] for a in s["Action"]] diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md index 11af0d657..b8a8b69c0 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md @@ -352,8 +352,19 @@ limited to: and the app's existing retrieve is denied. `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits the `BedrockStoredResponseOps` statement for a runtime `responses` row only and refuses for a `converse` / `messages` / `chat_completions` plan (`reason: stored_response_ops_no_responses_row`), which has no stored - responses. Verify with create-plus-retrieve: `r = client.responses.create(...)` then - `client.responses.retrieve(r.id)` under the attached role. + responses on either endpoint; a bare Mantle id on `responses` is covered by the `BedrockMantleInference` + statement (stored responses are retrievable on Mantle at `GET /v1/responses/{id}` under + `bedrock-mantle:CreateInference`, same page, read 2026-11-09) and gets no `bedrock:GetInvoke`. Verify with + create-plus-retrieve: `r = client.responses.create(...)` then `client.responses.retrieve(r.id)` under the + attached role. +6. Web Search → a row with `mantle_selector: web_search` needs `bedrock-websearch:InvokeSearch` and + `bedrock-websearch:InvokeFetch` on `*` (`iam_policy.py --mantle-selector web_search` emits them with an + `aws:RequestedRegion` condition); the client sets `"external_web_access": false` on the `web_search` tool + unless the user explicitly confirmed external access, in which case `--external-web-access` also grants + `bedrock-websearch:ExternalWebAccess` — the parameter defaults to `true` and every Fetch fails authorization + without that action (, + , read 2026-11-09). A plan + without a `web_search` row gets no `bedrock-websearch:*` action. **Mantle alternative (selector-gated).** Only when the row records a `mantle_selector` (`references/vendored/ai/bedrock-endpoint-policy.md`): base URL `https://bedrock-mantle.{region}.api.aws/openai/v1`, diff --git a/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py new file mode 100644 index 000000000..ae45632da --- /dev/null +++ b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py @@ -0,0 +1,175 @@ +"""Guards for the PR 588 review-round-2 findings that land on PR 585's branch (F12, F15, F3 Azure +half, F4 cloud half). + +Public basis (read 2026-11-09, all under https://docs.aws.amazon.com/bedrock/latest/userguide/): + inference-responses-api.html — stored responses exist on the Responses API of either endpoint; + bedrock-mantle inference authorizes `bedrock-mantle:CreateInference`, bedrock-runtime stored + response management authorizes `bedrock:GetInvoke` / `CancelInvoke` / `DeleteInvoke` on the + default project (F12). + web-search.html + security-web-search.html — actions `bedrock-websearch:InvokeSearch`, + `InvokeFetch`, `ExternalWebAccess` on resource `*`; `aws:RequestedRegion` the only condition + key; `external_web_access` defaults to true and Fetch fails without the action (F15). + model-card-openai-gpt-6-sol.html / -6-luna / -6-1-sol / -6-astra / -56-* / -55 / -54 — Guardrails + API scope is per model (F3). + web-search.html, inference-responses-api.html, projects.html, workspaces.html — the API each + selector requires (F4). + +Helper behaviour lives next to the helper (`skills/llm-to-bedrock/scripts/test_iam_policy_round2.py`, +classes TestStoredResponseOps / TestWebSearchGrants); this file pins the prose recipes, the +Design/schema contracts and the new design fixture. +""" +from __future__ import annotations + +import importlib.util +import json +import re +import sys +from pathlib import Path + +import pytest + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +SKILLS = PLUGIN_ROOT / "skills" +SCRIPTS = SKILLS / "llm-to-bedrock" / "scripts" +REPORT_GENERATOR = PLUGIN_ROOT / "agents" / "llm2bedrock-report-generator.md" +CODE_REWRITER = PLUGIN_ROOT / "agents" / "llm2bedrock-code-rewriter.md" +GCP_GENERATE_ARTIFACTS = SKILLS / "gcp-to-aws" / "references" / "phases" / "generate" / "generate-artifacts-ai.md" +AZURE_GENERATE_ARTIFACTS = SKILLS / "azure-to-aws" / "references" / "phases" / "generate" / "generate-artifacts-ai.md" +CANONICAL_GUIDE = SKILLS / "shared" / "ai" / "ai-openai-to-bedrock.md" +VENDORED_GUIDES = [ + SKILLS / "gcp-to-aws" / "references" / "vendored" / "ai" / "ai-openai-to-bedrock.md", + SKILLS / "azure-to-aws" / "references" / "vendored" / "ai" / "ai-openai-to-bedrock.md", +] +GCP_DESIGN_AI = SKILLS / "gcp-to-aws" / "references" / "phases" / "design" / "design-ai.md" +AZURE_DESIGN_AI = SKILLS / "azure-to-aws" / "references" / "phases" / "design" / "design-ai.md" +CANONICAL_SCHEMA_DOC = SKILLS / "shared" / "ai" / "schema-design-aws-ai.md" +VENDORED_SCHEMA_DOCS = [ + SKILLS / "gcp-to-aws" / "references" / "vendored" / "ai" / "schema-design-aws-ai.md", + SKILLS / "azure-to-aws" / "references" / "vendored" / "ai" / "schema-design-aws-ai.md", +] +GCP_SHADOW_SCHEMA = SKILLS / "gcp-to-aws" / "references" / "shared" / "schema-design-aws-ai.md" +AZURE_SHADOW_SCHEMA = SKILLS / "azure-to-aws" / "references" / "shared" / "schema-design-aws-ai.md" +ALL_SCHEMA_DOCS = [CANONICAL_SCHEMA_DOC, *VENDORED_SCHEMA_DOCS, GCP_SHADOW_SCHEMA, AZURE_SHADOW_SCHEMA] +DESIGN_FIXTURES = sorted((PLUGIN_ROOT / "fixtures" / "aws-design-ai").glob("*/aws-design-ai.json")) +NEW_FIXTURE = PLUGIN_ROOT / "fixtures" / "aws-design-ai" / "chat-source-web-search-continuity-kept" / "aws-design-ai.json" +DESIGN_AI_SCHEMA = PLUGIN_ROOT / "scripts" / "contracts" / "aws-design-ai.schema.json" +ARTIFACT_VALIDATOR = PLUGIN_ROOT / "scripts" / "validate-artifacts.py" + +RESPONSES_PAGE = "https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html" +WEB_SEARCH_PAGE = "https://docs.aws.amazon.com/bedrock/latest/userguide/web-search.html" +WEB_SEARCH_IAM_PAGE = "https://docs.aws.amazon.com/bedrock/latest/userguide/security-web-search.html" + + +def _load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) # type: ignore[union-attr] + return module + + +def _section(text: str, start: str, end: str) -> str: + i = text.index(start) + return text[i: text.index(end, i)] + + +def _squash(text: str) -> str: + return " ".join(text.split()) + + +def _recipe() -> tuple[str, str]: + raw = _section(REPORT_GENERATOR.read_text(), "# 6.4 Generate scoped IAM policy artifact", "# 7. Generate report") + return _squash(raw), _section(raw, "```bash", "```\n") + + +# --------------------------------------------------------------------------- F12 + F15: report §6.4 + +def test_report_recipe_passes_mantle_selectors_and_external_web_access_to_the_helper() -> None: + recipe, code = _recipe() + assert "[--mantle-selector None: + recipe, _ = _recipe() + para = _section(recipe, "**Retained stored-response operations need their own grant.**", "**Web Search needs its own grants.**") + assert "the grant is dispatched by endpoint" in para + assert "a `--target` row that is a bare Mantle id on `responses` (a `mantle_openai_responses` plan) satisfies retained ops through the `BedrockMantleInference` statement" in para + assert "`GET /v1/responses/{id}`" in para and "`bedrock-mantle:CreateInference`" in para + assert "no `bedrock:GetInvoke` is added for it and the helper does not exit 1" in para + assert "read 2026-11-09" in para + assert "Stored responses exist only on the runtime Responses API" not in recipe + + +def test_report_recipe_names_the_web_search_grants_and_the_fetch_default() -> None: + recipe, _ = _recipe() + para = _section(recipe, "**Web Search needs its own grants.**", "The script handles the dual-ARN pattern") + assert "`--mantle-selector web_search`" in para and "`BedrockWebSearch` statement" in para + assert "`bedrock-websearch:InvokeSearch`" in para and "`bedrock-websearch:InvokeFetch`" in para + assert "resource `*`" in para and "`aws:RequestedRegion` condition" in para + assert "`bedrock-websearch:ExternalWebAccess`" in para + assert 'MUST set `"external_web_access": false`' in para + assert "the parameter defaults to `true`" in para + assert "every Fetch then fails authorization before the cache is read" in para + assert "pass `--external-web-access` too" in para + assert WEB_SEARCH_PAGE in para and WEB_SEARCH_IAM_PAGE in para and "read 2026-11-09" in para + assert "`reason: web_search_without_mantle_row`" in para + assert "`reason: external_web_access_without_web_search`" in para + assert "A plan with no `web_search` row gets no `bedrock-websearch:*` action" in para + + +def test_report_checklist_has_a_tool_level_permission_check() -> None: + apply = _squash(_section(REPORT_GENERATOR.read_text(), "## How to Apply", "## How to Undo")) + assert "Tool-level permission check" in apply + assert "Render ONLY when a `bedrock_models[]` row has `mantle_selector: web_search`" in apply + assert 'tools=[{"type": "web_search", "external_web_access": False}]' in apply + assert "`web_search_call` item or a `url_citation` annotation" in apply + assert "CloudTrail Web Search data events for a denied `InvokeSearch` / `InvokeFetch`" in apply + assert "a plain model ping passing does not prove the Web Search grant" in apply + # The create-plus-retrieve bullet gained its Mantle variant. + assert "For a `mantle_openai_responses` plan run the same `responses.create` / `responses.retrieve(r.id)` pair against the Mantle base URL" in apply + assert "`BedrockMantleInference` statement is not attached" in apply + + +# --------------------------------------------------------------------------- F12 + F15: setup recipes + +@pytest.mark.parametrize("path", [GCP_GENERATE_ARTIFACTS, AZURE_GENERATE_ARTIFACTS, CANONICAL_GUIDE], ids=lambda p: p.parent.parent.name if "shared" not in p.parts else "canonical") +def test_setup_recipes_name_the_web_search_grants_and_mantle_stored_response_coverage(path: Path) -> None: + text = _squash(path.read_text()) + # Round-2 pinned strings survive. + assert "iam_policy.py --stored-response-ops retrieve[,cancel,delete]" in text + assert "client.responses.retrieve(r.id)" in text and "read 2026-11-08" in text + assert re.search(r"`converse`(,| /) `messages`", text), path + # F12 + assert "`BedrockMantleInference`" in text, path + assert "`GET /v1/responses/{id}`" in text and "`bedrock-mantle:CreateInference`" in text, path + assert "no `bedrock:GetInvoke`" in text, path + # F15 + assert "bedrock-websearch:InvokeSearch" in text and "bedrock-websearch:InvokeFetch" in text, path + assert "bedrock-websearch:ExternalWebAccess" in text, path + assert "--mantle-selector web_search" in text, path + assert '"external_web_access": false' in text, path + assert "defaults to `true`" in text, path + assert "--external-web-access" in text, path + assert WEB_SEARCH_PAGE in text and WEB_SEARCH_IAM_PAGE in text, path + assert "read 2026-11-09" in text, path + assert "no `bedrock-websearch:*` action" in text, path + + +def test_setup_verification_has_a_tool_level_web_search_check() -> None: + for path in (GCP_GENERATE_ARTIFACTS, AZURE_GENERATE_ARTIFACTS): + text = _squash(path.read_text()) + assert 'tools=[{"type": "web_search", "external_web_access": False}]' in text, path + assert "`web_search_call` item or a `url_citation` annotation" in text, path + assert "a plain model ping passing does not prove the grant" in text, path + + +def test_vendored_openai_guides_match_canonical() -> None: + for vendored in VENDORED_GUIDES: + assert vendored.read_bytes() == CANONICAL_GUIDE.read_bytes(), vendored + + From c764157444be813df9be78bb2291159375fd0ae7 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 9 Nov 2026 09:09:01 -0800 Subject: [PATCH 23/41] fix(aws-startup-advisor): Anthropic computer_/text_editor_/bash_ client tools are not server_side_tools signals (PR 588 F13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Sonnet 5 and Opus 5.5 model cards list `computer_20251124`, `computer_toolset_20260801`, `browser_toolset_20260801`, `bash_20250124`, `text_editor_20250728` and `memory_20250818` under "Computer use" and "Client tools and toolsets using `bedrock-runtime` and `bedrock-mantle` endpoints" (model-card-anthropic-claude-sonnet-5.html, model-card-anthropic-claude-opus-5-5.html, read 2026-11-09): they are client-executed and available on both endpoints, so they must not select Mantle. bedrock-endpoint-policy.md (canonical + 2 vendored mirrors via sync-vendored) and the gcp / azure / bedrock-migration-assessment discover-app-code.md scanners: the `server_side_tools` positive list keeps `"type": "mcp"`, `code_interpreter`, `file_search`, `computer_use_preview`, `image_generation` and Anthropic `code_execution_*` (provider-executed); `computer_*` / `text_editor_*` / `bash_*` (+ toolsets, `memory_*`) move to "Not a signal" with the card citation; both cards added to § Sources. New fixture fixtures/bma-scan-anthropic-client-tools (Anthropic SDK `claude-sonnet-5` with computer / text_editor / bash tools, profile with no `mantle_signals`). Tests (tests/test_mantle_selector_discovery.py): test_policy_and_scanners_keep_anthropic_client_tools_out_of_server_side_tools and test_client_tool_only_anthropic_fixture_has_no_mantle_signal_and_reaches_runtime_messages (validator ok, `mantle_signals_detected == []`, and iam_policy on `runtime_messages` emits `BedrockInvokeModelScoped` — the lane reaches IAM/preflight, not unsupported_lane). Validation: plugin `pytest tests -q` 1532 passed, 1 failed (known test_model_id_lint::test_real_plugin_passes .venv noise); scripts/validate-artifacts.py --fixtures PASS (103); tools/sync-vendored.py --check PASS (64). --- .../ai-workload-profile.json | 16 ++++ .../repo/app/agent.py | 18 +++++ .../repo/requirements.txt | 1 + .../phases/discover/discover-app-code.md | 6 +- .../vendored/ai/bedrock-endpoint-policy.md | 13 +++- .../phases/discover/discover-app-code.md | 2 +- .../phases/discover/discover-app-code.md | 2 +- .../vendored/ai/bedrock-endpoint-policy.md | 13 +++- .../shared/ai/bedrock-endpoint-policy.md | 13 +++- .../tests/test_mantle_selector_discovery.py | 74 +++++++++++++++++++ 10 files changed, 141 insertions(+), 17 deletions(-) create mode 100644 plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/ai-workload-profile.json create mode 100644 plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/repo/app/agent.py create mode 100644 plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/repo/requirements.txt diff --git a/plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/ai-workload-profile.json b/plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/ai-workload-profile.json new file mode 100644 index 000000000..005e013c3 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/ai-workload-profile.json @@ -0,0 +1,16 @@ +{ + "schema_version": "1.0", + "ai_sources": [ + { + "provider": "anthropic", + "source_service": "anthropic_api", + "model_families": ["claude"], + "model_evidence": "observed", + "models": ["claude-sonnet-5"], + "model_observations": [ + {"normalized_model": "claude-sonnet-5", "raw_model": "claude-sonnet-5", "model_family": "claude", "evidence": [{"provenance": "code", "confidence": "high", "detail": "app/agent.py:8 model=\"claude-sonnet-5\" in client.beta.messages.create"}]} + ] + } + ], + "gateways": [] +} diff --git a/plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/repo/app/agent.py b/plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/repo/app/agent.py new file mode 100644 index 000000000..3a780cf7d --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/repo/app/agent.py @@ -0,0 +1,18 @@ +import anthropic + +client = anthropic.Anthropic() + + +def run_agent(task: str) -> str: + resp = client.beta.messages.create( + model="claude-sonnet-5", + max_tokens=2048, + tools=[ + {"type": "computer_20251124", "name": "computer", "display_width_px": 1280, "display_height_px": 800}, + {"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"}, + {"type": "bash_20250124", "name": "bash"}, + ], + betas=["computer-use-2025-11-24"], + messages=[{"role": "user", "content": task}], + ) + return "".join(block.text for block in resp.content if block.type == "text") diff --git a/plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/repo/requirements.txt b/plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/repo/requirements.txt new file mode 100644 index 000000000..4d3988d90 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/repo/requirements.txt @@ -0,0 +1 @@ +anthropic>=0.40 diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-app-code.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-app-code.md index e9fb60de4..fc2da023e 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-app-code.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/discover/discover-app-code.md @@ -186,9 +186,9 @@ call arguments, the client construction it uses, and request headers. Azure Open merely named `web_search`. - `server_side_tools` — a tools entry of `"type": "mcp"` (`server_url` / `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, or `image_generation` used as a tool type; - Anthropic `code_execution_*` / `text_editor_*` / `computer_*` / `bash_*` tool types — any - provider-executed tool other than web search. NOT: `"type": "function"` tools, Gemini - `function_declarations=`, `@tool` decorators, `tool_choice` — client-side function tools never count. + Anthropic `code_execution_*` tool types — any provider-executed tool other than web search. NOT: + `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, `tool_choice` — + client-side function tools never count; Anthropic `computer_*` (incl. `computer_toolset_*`), `browser_toolset_*`, `text_editor_*`, `bash_*` and `memory_*` are CLIENT-executed tools listed on both `bedrock-runtime` and `bedrock-mantle` by the Sonnet 5 and Opus 5.5 cards (`model-card-anthropic-claude-sonnet-5.html`, `model-card-anthropic-claude-opus-5-5.html`, read 2026-11-09) — a client-tool-only Anthropic app stays `runtime_messages`. - `projects` — `OpenAI-Project` header (`default_headers={"OpenAI-Project": ...}`), `project=` kwarg on the OpenAI client, or an `OPENAI_PROJECT_ID`-style env name read in code. NOT: `OpenAI-Organization` alone; no header means the default project. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md index eb1331580..4f1771cfa 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/bedrock-endpoint-policy.md @@ -82,10 +82,10 @@ The seven values are learned in three different ways; no phase may guess one it client-side `"type": "function"` tool merely named `web_search`. - `server_side_tools` — positive: a tools entry of `"type": "mcp"` (`server_url` / `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, - `image_generation` used as a tool type, or an Anthropic `code_execution_*` / `text_editor_*` - / `computer_*` / `bash_*` tool type — any provider-executed tool other than web search. Not a - signal: `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, - `tool_choice` — client-side function tools never count. + `image_generation` used as a tool type, or an Anthropic `code_execution_*` tool type — any + provider-executed tool other than web search. Not a signal: `"type": "function"` tools, + Gemini `function_declarations=`, `@tool` decorators, `tool_choice` — client-side function + tools never count; Anthropic `computer_*` (incl. `computer_toolset_*`), `browser_toolset_*`, `text_editor_*`, `bash_*` and `memory_*` are CLIENT-executed tools listed on both `bedrock-runtime` and `bedrock-mantle` by the Sonnet 5 and Opus 5.5 cards (`model-card-anthropic-claude-sonnet-5.html`, `model-card-anthropic-claude-opus-5-5.html`, read 2026-11-09) — a client-tool-only Anthropic app stays `runtime_messages`. - `projects` — positive: an `OpenAI-Project` header (`default_headers={"OpenAI-Project": ...}`), a `project=` kwarg on the OpenAI client, or an `OPENAI_PROJECT_ID`-style environment name read in code; in usage discovery, a @@ -153,6 +153,11 @@ All pages under `https://docs.aws.amazon.com/bedrock/latest/userguide/`, read 20 - `model-card-openai-gpt-54.html` and `model-card-openai-gpt-55.html` (read 2026-11-08) — commercial In-Region prices only. Runtime lists `us.` and `global.` ids, and those rows publish no price. +- `model-card-anthropic-claude-sonnet-5.html` and `model-card-anthropic-claude-opus-5-5.html` + (read 2026-11-09) — "Computer use" and "Client tools and toolsets using `bedrock-runtime` and + `bedrock-mantle` endpoints": `computer_20251124`, `computer_toolset_20260801`, + `browser_toolset_20260801`, `bash_20250124`, `text_editor_20250728`, `memory_20250818` are + client-executed on both endpoints. - `models-endpoint-availability.html` - `inference-responses-api.html` - `inference-chat-completions.html` diff --git a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/discover/discover-app-code.md b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/discover/discover-app-code.md index 8e58dc8d8..eb7b6ac25 100644 --- a/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/discover/discover-app-code.md +++ b/plugins/aws-startup-advisor/skills/bedrock-migration-assessment/references/phases/discover/discover-app-code.md @@ -200,7 +200,7 @@ A few provider capabilities decide the target endpoint later (`skills/shared/ai/ | ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | | `background_inference` | `background=True` / `background: true` on `responses.create`, or a raw `"background": true` key in a Responses request body | `store=True`, `stream=True`, polling `responses.retrieve` without `background` | | `web_search` | `tools=[{"type": "web_search"}]`; `"type": "web_search_preview"` (any dated `web_search_preview_*` form); Anthropic `"type": "web_search_20250305"`; OpenRouter `plugins: [{"id": "web"}]` or a `:online` model suffix | A client-side `"type": "function"` tool merely NAMED `web_search` | -| `server_side_tools` | A tools entry of `"type": "mcp"` (`server_url` / `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, or `image_generation` used as a tool type; Anthropic `code_execution_*` / `text_editor_*` / `computer_*` / `bash_*` tool types — any provider-executed tool other than web search | `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, `tool_choice` — client-side function tools never count | +| `server_side_tools` | A tools entry of `"type": "mcp"` (`server_url` / `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, or `image_generation` used as a tool type; Anthropic `code_execution_*` tool types — any provider-executed tool other than web search | `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, `tool_choice` — client-side function tools never count; Anthropic `computer_*` (incl. `computer_toolset_*`), `browser_toolset_*`, `text_editor_*`, `bash_*` and `memory_*` are CLIENT-executed tools listed on both `bedrock-runtime` and `bedrock-mantle` by the Sonnet 5 and Opus 5.5 cards (`model-card-anthropic-claude-sonnet-5.html`, `model-card-anthropic-claude-opus-5-5.html`, read 2026-11-09) — a client-tool-only Anthropic app stays `runtime_messages` | | `projects` | `OpenAI-Project` header (`default_headers={"OpenAI-Project": ...}`), `project=` kwarg on the OpenAI client, or an `OPENAI_PROJECT_ID`-style environment variable name read in code | `OpenAI-Organization` alone; no header means the default project | | `workspaces` | `anthropic-workspace-id` header or an `ANTHROPIC_WORKSPACE_ID`-style environment variable name | OpenRouter's own `workspace_id` scoping (an OpenRouter account concept) | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-app-code.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-app-code.md index 395a1f892..4f7ffe7c5 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-app-code.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/discover/discover-app-code.md @@ -396,7 +396,7 @@ A few provider capabilities decide the target endpoint later (`references/vendor | ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | | `background_inference` | `background=True` / `background: true` on `responses.create`, or a raw `"background": true` key in a Responses request body | `store=True`, `stream=True`, polling `responses.retrieve` without `background` | | `web_search` | `tools=[{"type": "web_search"}]`; `"type": "web_search_preview"` (any dated `web_search_preview_*` form); Anthropic `"type": "web_search_20250305"`; OpenRouter `plugins: [{"id": "web"}]` or a `:online` model suffix | A client-side `"type": "function"` tool merely NAMED `web_search` | -| `server_side_tools` | A tools entry of `"type": "mcp"` (`server_url` / `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, or `image_generation` used as a tool type; Anthropic `code_execution_*` / `text_editor_*` / `computer_*` / `bash_*` tool types — any provider-executed tool other than web search | `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, `tool_choice` — client-side function tools never count | +| `server_side_tools` | A tools entry of `"type": "mcp"` (`server_url` / `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, or `image_generation` used as a tool type; Anthropic `code_execution_*` tool types — any provider-executed tool other than web search | `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, `tool_choice` — client-side function tools never count; Anthropic `computer_*` (incl. `computer_toolset_*`), `browser_toolset_*`, `text_editor_*`, `bash_*` and `memory_*` are CLIENT-executed tools listed on both `bedrock-runtime` and `bedrock-mantle` by the Sonnet 5 and Opus 5.5 cards (`model-card-anthropic-claude-sonnet-5.html`, `model-card-anthropic-claude-opus-5-5.html`, read 2026-11-09) — a client-tool-only Anthropic app stays `runtime_messages` | | `projects` | `OpenAI-Project` header (`default_headers={"OpenAI-Project": ...}`), `project=` kwarg on the OpenAI client, or an `OPENAI_PROJECT_ID`-style env name read in code | `OpenAI-Organization` alone; no header means the default project | | `workspaces` | `anthropic-workspace-id` header or an `ANTHROPIC_WORKSPACE_ID`-style env name | OpenRouter's own `workspace_id` scoping (an OpenRouter account concept) | diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md index eb1331580..4f1771cfa 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/bedrock-endpoint-policy.md @@ -82,10 +82,10 @@ The seven values are learned in three different ways; no phase may guess one it client-side `"type": "function"` tool merely named `web_search`. - `server_side_tools` — positive: a tools entry of `"type": "mcp"` (`server_url` / `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, - `image_generation` used as a tool type, or an Anthropic `code_execution_*` / `text_editor_*` - / `computer_*` / `bash_*` tool type — any provider-executed tool other than web search. Not a - signal: `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, - `tool_choice` — client-side function tools never count. + `image_generation` used as a tool type, or an Anthropic `code_execution_*` tool type — any + provider-executed tool other than web search. Not a signal: `"type": "function"` tools, + Gemini `function_declarations=`, `@tool` decorators, `tool_choice` — client-side function + tools never count; Anthropic `computer_*` (incl. `computer_toolset_*`), `browser_toolset_*`, `text_editor_*`, `bash_*` and `memory_*` are CLIENT-executed tools listed on both `bedrock-runtime` and `bedrock-mantle` by the Sonnet 5 and Opus 5.5 cards (`model-card-anthropic-claude-sonnet-5.html`, `model-card-anthropic-claude-opus-5-5.html`, read 2026-11-09) — a client-tool-only Anthropic app stays `runtime_messages`. - `projects` — positive: an `OpenAI-Project` header (`default_headers={"OpenAI-Project": ...}`), a `project=` kwarg on the OpenAI client, or an `OPENAI_PROJECT_ID`-style environment name read in code; in usage discovery, a @@ -153,6 +153,11 @@ All pages under `https://docs.aws.amazon.com/bedrock/latest/userguide/`, read 20 - `model-card-openai-gpt-54.html` and `model-card-openai-gpt-55.html` (read 2026-11-08) — commercial In-Region prices only. Runtime lists `us.` and `global.` ids, and those rows publish no price. +- `model-card-anthropic-claude-sonnet-5.html` and `model-card-anthropic-claude-opus-5-5.html` + (read 2026-11-09) — "Computer use" and "Client tools and toolsets using `bedrock-runtime` and + `bedrock-mantle` endpoints": `computer_20251124`, `computer_toolset_20260801`, + `browser_toolset_20260801`, `bash_20250124`, `text_editor_20250728`, `memory_20250818` are + client-executed on both endpoints. - `models-endpoint-availability.html` - `inference-responses-api.html` - `inference-chat-completions.html` diff --git a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md index eb1331580..4f1771cfa 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/bedrock-endpoint-policy.md @@ -82,10 +82,10 @@ The seven values are learned in three different ways; no phase may guess one it client-side `"type": "function"` tool merely named `web_search`. - `server_side_tools` — positive: a tools entry of `"type": "mcp"` (`server_url` / `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, - `image_generation` used as a tool type, or an Anthropic `code_execution_*` / `text_editor_*` - / `computer_*` / `bash_*` tool type — any provider-executed tool other than web search. Not a - signal: `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, - `tool_choice` — client-side function tools never count. + `image_generation` used as a tool type, or an Anthropic `code_execution_*` tool type — any + provider-executed tool other than web search. Not a signal: `"type": "function"` tools, + Gemini `function_declarations=`, `@tool` decorators, `tool_choice` — client-side function + tools never count; Anthropic `computer_*` (incl. `computer_toolset_*`), `browser_toolset_*`, `text_editor_*`, `bash_*` and `memory_*` are CLIENT-executed tools listed on both `bedrock-runtime` and `bedrock-mantle` by the Sonnet 5 and Opus 5.5 cards (`model-card-anthropic-claude-sonnet-5.html`, `model-card-anthropic-claude-opus-5-5.html`, read 2026-11-09) — a client-tool-only Anthropic app stays `runtime_messages`. - `projects` — positive: an `OpenAI-Project` header (`default_headers={"OpenAI-Project": ...}`), a `project=` kwarg on the OpenAI client, or an `OPENAI_PROJECT_ID`-style environment name read in code; in usage discovery, a @@ -153,6 +153,11 @@ All pages under `https://docs.aws.amazon.com/bedrock/latest/userguide/`, read 20 - `model-card-openai-gpt-54.html` and `model-card-openai-gpt-55.html` (read 2026-11-08) — commercial In-Region prices only. Runtime lists `us.` and `global.` ids, and those rows publish no price. +- `model-card-anthropic-claude-sonnet-5.html` and `model-card-anthropic-claude-opus-5-5.html` + (read 2026-11-09) — "Computer use" and "Client tools and toolsets using `bedrock-runtime` and + `bedrock-mantle` endpoints": `computer_20251124`, `computer_toolset_20260801`, + `browser_toolset_20260801`, `bash_20250124`, `text_editor_20250728`, `memory_20250818` are + client-executed on both endpoints. - `models-endpoint-availability.html` - `inference-responses-api.html` - `inference-chat-completions.html` diff --git a/plugins/aws-startup-advisor/tests/test_mantle_selector_discovery.py b/plugins/aws-startup-advisor/tests/test_mantle_selector_discovery.py index d5290a7ab..39da9543b 100644 --- a/plugins/aws-startup-advisor/tests/test_mantle_selector_discovery.py +++ b/plugins/aws-startup-advisor/tests/test_mantle_selector_discovery.py @@ -183,3 +183,77 @@ def test_summary_lists_detected_signals() -> None: assert item["detail"].startswith("app/llm.py:") negative = json.loads((NEGATIVE_FIXTURE / "ai-workload-profile.json").read_text(encoding="utf-8")) assert VALIDATOR.validate_profile(negative)["mantle_signals_detected"] == [] + + +# --------------------------------------------------------------------------- (h) PR 588 F13: client tools +# Anthropic computer_* / text_editor_* / bash_* (and computer_toolset_* / browser_toolset_* / memory_*) are +# CLIENT-executed tools the Sonnet 5 and Opus 5.5 cards list on both bedrock-runtime and bedrock-mantle +# (model-card-anthropic-claude-sonnet-5.html, model-card-anthropic-claude-opus-5-5.html, read 2026-11-09). +# They must not produce a server_side_tools mantle_signal; only provider-executed tools do. +CLIENT_TOOL_FIXTURE = PLUGIN_ROOT / "fixtures" / "bma-scan-anthropic-client-tools" +CLIENT_TOOL_PATTERNS = ["`computer_*`", "`text_editor_*`", "`bash_*`"] + + +def _server_side_tools_entry(text: str, name: str) -> tuple[str, str]: + """Return (positive, negative) halves of the server_side_tools entry in a scanner or the policy.""" + if name in ("gcp", "bma"): + row = next(line for line in text.splitlines() if line.startswith("| `server_side_tools`")) + cells = [c.strip() for c in row.strip().strip("|").split("|")] + return cells[1], cells[2] + marker = "`server_side_tools` — " + start = text.index(marker) + len(marker) + end = text.index("`projects` — ", start) + entry = " ".join(text[start:end].split()) + split = re.search(r"\b(NOT|Not a signal):", entry) + return entry[:split.start()], entry[split.start():] + + +def test_policy_and_scanners_keep_anthropic_client_tools_out_of_server_side_tools() -> None: + targets = {**SCANNERS, "policy": POLICY_CANONICAL, + **{f"policy-vendored-{i}": p for i, p in enumerate(POLICY_VENDORED)}} + for name, path in targets.items(): + text = path.read_text(encoding="utf-8") + kind = name if name in ("gcp", "bma") else "azure" + positive, negative = _server_side_tools_entry(text, kind) + assert "`code_execution_*`" in positive, f"{name}: provider-executed code_execution_* must stay positive" + assert '"type": "mcp"' in positive, name + for pattern in CLIENT_TOOL_PATTERNS: + assert pattern not in positive, f"{name}: {pattern} is client-executed and must not be a positive signal" + assert pattern in negative, f"{name}: {pattern} must be listed as not-a-signal" + assert "CLIENT-executed" in negative and "`bedrock-runtime` and `bedrock-mantle`" in negative, name + assert "model-card-anthropic-claude-sonnet-5.html" in negative, name + assert "model-card-anthropic-claude-opus-5-5.html" in negative, name + assert "read 2026-11-09" in negative, name + assert "stays `runtime_messages`" in negative, name + policy = POLICY_CANONICAL.read_text(encoding="utf-8") + sources = policy[policy.index("## Sources"):] + assert "model-card-anthropic-claude-sonnet-5.html" in sources and "model-card-anthropic-claude-opus-5-5.html" in sources + + +def test_client_tool_only_anthropic_fixture_has_no_mantle_signal_and_reaches_runtime_messages() -> None: + profile = json.loads((CLIENT_TOOL_FIXTURE / "ai-workload-profile.json").read_text(encoding="utf-8")) + result = VALIDATOR.validate_profile(profile) + assert result["ok"] is True + assert result["mantle_signals_detected"] == [] + assert "mantle_signals" not in profile + source = (CLIENT_TOOL_FIXTURE / "repo" / "app" / "agent.py").read_text(encoding="utf-8") + for tool_type in ('"type": "computer_20251124"', '"type": "text_editor_20250728"', '"type": "bash_20250124"'): + assert tool_type in source, tool_type + assert '"type": "mcp"' not in source and "web_search" not in source and "code_execution" not in source + for observation in profile["ai_sources"][0]["model_observations"]: + for entry in observation["evidence"]: + location = entry["detail"].split(" ", 1)[0] + path, line = location.split(":") + assert 'model="claude-sonnet-5"' in (CLIENT_TOOL_FIXTURE / "repo" / path).read_text( + encoding="utf-8").splitlines()[int(line) - 1] + # The runtime Messages lane reaches IAM / preflight instead of an unsupported-lane stop. + iam_spec = importlib.util.spec_from_file_location( + "iam_policy_client_tools", SKILLS / "llm-to-bedrock" / "scripts" / "iam_policy.py") + iam = importlib.util.module_from_spec(iam_spec) + iam_spec.loader.exec_module(iam) + claude = "global.anthropic.claude-sonnet-5" + policy = iam.generate_policy([claude], "us-east-1", "123456789012", migration_path="runtime_messages", + target_apis={claude: "messages"}) + sids = [s["Sid"] for s in policy["Statement"]] + assert "BedrockInvokeModelScoped" in sids + assert not any(sid.startswith("BedrockMantle") for sid in sids) From afcd9834ad5805b8a572ff56cd96f49c5ae3bfa0 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 9 Nov 2026 09:09:43 -0800 Subject: [PATCH 24/41] docs(aws-startup-advisor): Azure Design/schema and the code-rewriter carry the per-model Guardrails API rule (PR 588 F3, Azure half) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The blanket "`converse` / `chat_completions` / `invoke` (never `responses`) for 6.1 Sol and the other GPT-6/5.6 CRIS models" in skills/azure-to-aws design-ai.md (Part 2 Guardrails block) and references/shared/schema-design-aws-ai.md (§ required_client_api) is replaced with the per-model rule GCP already carries on #588: `converse` for GPT-6 Astra, GPT-5.6 Sol / Terra / Luna, GPT-5.5, GPT-5.4 and GPT-6.1 Terra (Converse-only); `responses` / `chat_completions` / `invoke` / `converse` for GPT-6 Sol and GPT-6 Luna; `chat_completions` / `invoke` / `converse` (never `responses`) for GPT-6.1 Sol; the nine card URLs are listed and "Do not leave the client on `responses` when Guardrails is required on a model whose card excludes Responses." closes the block. agents/llm2bedrock-code-rewriter.md: "note Guardrails are Converse-only and prompt caching Responses-only on runtime" becomes the per-card scope carried by `bedrock_models[].required_client_api` (GPT-6 Sol / Luna on Responses, Chat Completions, InvokeModel and Converse; GPT-6.1 Sol not on Responses; GPT-5.6 / 5.5 / 5.4 / Astra Converse only; implicit caching Responses-only on runtime for GPT-5.5 / 5.4, broader for 5.6 / 6). Public basis, read 2026-11-09: model-card-openai-gpt-6-sol.html "Guardrails (Responses, Chat Completions, InvokeModel, and Converse APIs)"; -6-luna same; -6-1-sol "Chat Completions, InvokeModel, and Converse APIs; not supported with Responses"; -6-astra, -56-sol/-terra/-luna, -55, -54 "Converse API only". The canonical and GCP shadow schema docs contain no blanket wording (grep verified); the GCP design block is replaced by #588's ea0798b3 when it rebases. Tests (tests/test_runtime_default_review_round3.py): blanket strings absent in Azure design, Azure/canonical/GCP schema docs and the rewriter; per-model lines + nine cards present in the Azure design and schema; rewriter sentence pinned. Validation: plugin `pytest tests -q` 1532 passed, 1 failed (known model-id-lint .venv noise); markdownlint-cli2 0 issues. --- .../agents/llm2bedrock-code-rewriter.md | 2 +- .../references/phases/design/design-ai.md | 12 +++-- .../references/shared/schema-design-aws-ai.md | 11 ++-- .../test_runtime_default_review_round3.py | 51 +++++++++++++++++++ 4 files changed, 69 insertions(+), 7 deletions(-) diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md b/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md index fe05d5eac..82eaac4a2 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-code-rewriter.md @@ -315,7 +315,7 @@ client = anthropic.Anthropic( Preserve compatible SDK request/response shapes. A model version change still requires §9 target checks, including typed response parsing for Opus 5.5. Apply the Chat Completions → Responses reshape only when that API change was selected. Preserve the exact validated runtime profile ID or ARN. After applying the changes above, skip the Converse-specific guidance in the rest of §8; the §9 behavior-delta application still applies normally. -**Converse for a proprietary GPT target is id-form-dependent, not family-dependent.** A **bare** proprietary GPT id (`openai.gpt-5.5`, `openai.gpt-5.4`, `openai.gpt-5.6-*`, `openai.gpt-6-*`) is a `bedrock-mantle` id with no `bedrock-runtime` in-region form — a boto3 `converse()` call against a bare id fails at runtime; if your context pairs a bare id with the Converse path, stop and report the contradiction. GPT-5.6, GPT-6 (Astra, Sol, Luna, 6.1 Sol) **and GPT-5.5 / GPT-5.4** (model cards read 2026-11-06: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`; Responses, Chat Completions and Converse on runtime, no Invoke) all have a `bedrock-runtime` path via CRIS ids (`us.`/`in.`/`global.` prefixed for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4 — those cards publish no `in.` profile): when the plan says `migration_path: runtime_openai_cris`, a Converse, Responses or Chat Completions rewrite against the CRIS id is valid (base URL `bedrock-runtime.{region}.amazonaws.com/openai/v1` for the OpenAI-compatible APIs; note Guardrails are Converse-only and prompt caching Responses-only on runtime). The Runtime OpenAI lane is the default for a compatible source; use the Mantle express lane only when the plan recorded a `mantle_*` path with a `mantle_selector`. +**Converse for a proprietary GPT target is id-form-dependent, not family-dependent.** A **bare** proprietary GPT id (`openai.gpt-5.5`, `openai.gpt-5.4`, `openai.gpt-5.6-*`, `openai.gpt-6-*`) is a `bedrock-mantle` id with no `bedrock-runtime` in-region form — a boto3 `converse()` call against a bare id fails at runtime; if your context pairs a bare id with the Converse path, stop and report the contradiction. GPT-5.6, GPT-6 (Astra, Sol, Luna, 6.1 Sol) **and GPT-5.5 / GPT-5.4** (model cards read 2026-11-06: `us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`; Responses, Chat Completions and Converse on runtime, no Invoke) all have a `bedrock-runtime` path via CRIS ids (`us.`/`in.`/`global.` prefixed for GPT-5.6 Terra and Luna; `us.`/`global.` only for GPT-5.6 Sol, GPT-6, GPT-5.5 and GPT-5.4 — those cards publish no `in.` profile): when the plan says `migration_path: runtime_openai_cris`, a Converse, Responses or Chat Completions rewrite against the CRIS id is valid (base URL `bedrock-runtime.{region}.amazonaws.com/openai/v1` for the OpenAI-compatible APIs; Guardrails and prompt-caching API scope are per model card and are carried by `bedrock_models[].required_client_api`: GPT-6 Sol / Luna list Guardrails on Responses, Chat Completions, InvokeModel and Converse; GPT-6.1 Sol on Chat Completions, InvokeModel and Converse (not Responses); GPT-5.6 Sol / Terra / Luna, GPT-5.5, GPT-5.4 and GPT-6 Astra on Converse only; implicit prompt caching is Responses-only on runtime for GPT-5.5 / GPT-5.4 and on Responses, Chat Completions and Converse for GPT-5.6 / GPT-6 — cards read 2026-11-09). The Runtime OpenAI lane is the default for a compatible source; use the Mantle express lane only when the plan recorded a `mantle_*` path with a `mantle_selector`. ### Converse rewrite (explicit `converse` plans) diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md index 5fb73620b..f0e7352f4 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md @@ -289,9 +289,15 @@ for GPT-6 Astra is valid only via Responses/Chat Completions/Converse — Astra Sol/Luna/6.1 Sol. **When Guardrails is the driving need and the design keeps `runtime_openai_cris` (endpoint switch, not model switch), it MUST also record `bedrock_models[].required_client_api` so the generated client uses a Guardrails-capable -surface — `converse` for Astra and 6.1 Terra; `converse` / `chat_completions` / `invoke` (never `responses`) -for 6.1 Sol and the other GPT-6/5.6 CRIS models except Astra and 6.1 Terra. Do not leave the client on -`responses` when Guardrails is required.** For a region-only CRIS move with no Guardrails or +surface — `converse` for GPT-6 Astra, GPT-5.6 Sol / Terra / Luna, GPT-5.5, GPT-5.4 and GPT-6.1 Terra +(their cards list Guardrails as Converse-only); `responses` / `chat_completions` / `invoke` / `converse` +for GPT-6 Sol and GPT-6 Luna; `chat_completions` / `invoke` / `converse` (never `responses`) for GPT-6.1 +Sol — model cards read 2026-11-09 (`model-card-openai-gpt-6-sol.html`, `model-card-openai-gpt-6-luna.html`, +`model-card-openai-gpt-6-1-sol.html`, `model-card-openai-gpt-6-astra.html`, +`model-card-openai-gpt-56-sol.html`, `model-card-openai-gpt-56-terra.html`, +`model-card-openai-gpt-56-luna.html`, `model-card-openai-gpt-55.html`, `model-card-openai-gpt-54.html` +under `https://docs.aws.amazon.com/bedrock/latest/userguide/`). Do not leave the client on +`responses` when Guardrails is required on a model whose card excludes Responses.** For a region-only CRIS move with no Guardrails or application-inference-profile need, the default `required_client_api` is the API the source app already calls (`responses` or `chat_completions`) — never an implicit Converse. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md index 4a315d67d..dbb5037fb 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md @@ -192,9 +192,14 @@ client-surface need) record the API the source app already calls — `responses` source, `chat_completions` for a Chat Completions source; Converse is never an implicit default for `runtime_openai_cris`. `messages` belongs to `runtime_messages`. A Guardrails-requiring workload on `runtime_openai_cris` sets it to the surface that is actually Guardrails-capable for that model: -`converse` for GPT-6 Astra and GPT-6.1 Terra, or -`converse`/`chat_completions`/`invoke` — never `responses` — for GPT-6.1 Sol and the other -GPT-6/5.6 CRIS models except Astra and GPT-6.1 Terra. Application inference profiles also require +`converse` for GPT-6 Astra, GPT-5.6 Sol / Terra / Luna, GPT-5.5, GPT-5.4 and GPT-6.1 Terra (their +cards list Guardrails as Converse-only); `responses` / `chat_completions` / `invoke` / `converse` for +GPT-6 Sol and GPT-6 Luna; `chat_completions` / `invoke` / `converse` (never `responses`) for GPT-6.1 +Sol — the Guardrails API scope is per model card, model cards read 2026-11-09 (`model-card-openai-gpt-6-sol.html`, `model-card-openai-gpt-6-luna.html`, +`model-card-openai-gpt-6-1-sol.html`, `model-card-openai-gpt-6-astra.html`, +`model-card-openai-gpt-56-sol.html`, `model-card-openai-gpt-56-terra.html`, +`model-card-openai-gpt-56-luna.html`, `model-card-openai-gpt-55.html`, `model-card-openai-gpt-54.html` +under `https://docs.aws.amazon.com/bedrock/latest/userguide/`). Application inference profiles also require `converse` on every model whose card lists them as Converse-only — GPT-6.1 Terra, Astra, and GPT-5.5 / GPT-5.4 (cards read 2026-11-07: "Converse API only; not supported with Responses or Chat Completions APIs") — so a workload needing them on `runtime_openai_cris` sets `converse`, never leaves the field diff --git a/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py index ae45632da..92a999ab3 100644 --- a/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py +++ b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py @@ -173,3 +173,54 @@ def test_vendored_openai_guides_match_canonical() -> None: assert vendored.read_bytes() == CANONICAL_GUIDE.read_bytes(), vendored +# --------------------------------------------------------------------------- F3 (Azure half): per-model Guardrails rule + +BLANKET_GUARDRAILS = [ + "never `responses`) for 6.1 Sol and the other GPT-6/5.6", + "the other GPT-6/5.6 CRIS models except Astra", + "note Guardrails are Converse-only", +] +NINE_CARDS = [ + "model-card-openai-gpt-6-sol.html", "model-card-openai-gpt-6-luna.html", "model-card-openai-gpt-6-1-sol.html", + "model-card-openai-gpt-6-astra.html", "model-card-openai-gpt-56-sol.html", "model-card-openai-gpt-56-terra.html", + "model-card-openai-gpt-56-luna.html", "model-card-openai-gpt-55.html", "model-card-openai-gpt-54.html", +] + + +@pytest.mark.parametrize("path", [AZURE_DESIGN_AI, AZURE_SHADOW_SCHEMA, CANONICAL_SCHEMA_DOC, GCP_SHADOW_SCHEMA, CODE_REWRITER], + ids=lambda p: p.name if p.suffix == ".md" and "agents" in p.parts else str(p.relative_to(SKILLS))) +def test_blanket_converse_only_guardrails_wording_is_gone(path: Path) -> None: + text = _squash(path.read_text()) + for blanket in BLANKET_GUARDRAILS: + assert blanket not in text, f"{path}: blanket wording {blanket!r} still present" + + +@pytest.mark.parametrize("path", [AZURE_DESIGN_AI, AZURE_SHADOW_SCHEMA], ids=lambda p: p.parent.name) +def test_azure_design_and_schema_carry_the_per_model_guardrails_api_rule(path: Path) -> None: + text = _squash(path.read_text()) + assert "`converse` for GPT-6 Astra, GPT-5.6 Sol / Terra / Luna, GPT-5.5, GPT-5.4 and GPT-6.1 Terra" in text + assert "`responses` / `chat_completions` / `invoke` / `converse` for GPT-6 Sol and GPT-6 Luna" in text + assert "`chat_completions` / `invoke` / `converse` (never `responses`) for GPT-6.1 Sol" in text + assert "model cards read 2026-11-09" in text + for card in NINE_CARDS: + assert card in text, f"{path}: {card} not cited" + + +def test_azure_design_closes_with_the_card_scoped_responses_rule() -> None: + azure = _squash(AZURE_DESIGN_AI.read_text()) + assert "Do not leave the client on `responses` when Guardrails is required on a model whose card excludes Responses." in azure + assert "Do not leave the client on `responses` when Guardrails is required.**" not in azure + for name in ("GPT-6 Sol", "GPT-6 Luna", "GPT-6.1 Sol", "GPT-6 Astra", "GPT-5.5", "GPT-5.4", "GPT-6.1 Terra"): + assert name in azure, name + + +def test_code_rewriter_carries_the_per_model_guardrails_and_caching_scope() -> None: + text = _squash(CODE_REWRITER.read_text()) + assert "Guardrails and prompt-caching API scope are per model card and are carried by `bedrock_models[].required_client_api`" in text + assert "GPT-6 Sol / Luna list Guardrails on Responses, Chat Completions, InvokeModel and Converse" in text + assert "GPT-6.1 Sol on Chat Completions, InvokeModel and Converse (not Responses)" in text + assert "GPT-5.6 Sol / Terra / Luna, GPT-5.5, GPT-5.4 and GPT-6 Astra on Converse only" in text + assert "implicit prompt caching is Responses-only on runtime for GPT-5.5 / GPT-5.4" in text + assert "cards read 2026-11-09" in text + + From a8a0ec3062e5fffe1f69dbfdd15936f2f0445d84 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 9 Nov 2026 09:10:07 -0800 Subject: [PATCH 25/41] fix(aws-startup-advisor): Step 0.8 validates a matched selector against its required API; Responses-only selectors on a Chat/Messages source are an explicit decision (PR 588 F4, cloud half) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GCP and Azure design-ai.md Step 0.8 gain rule 1b between rules 1 and 2, and rule 2 now reads "Any match that passes 1b": `web_search` and `background_inference` require the Responses API (web-search.html; inference-responses-api.html, read 2026-11-09); `workspaces` requires the Anthropic Messages API (`anthropic-workspace-id`); `projects` requires an OpenAI-compatible API (`OpenAI-Project`). A provider mismatch (`workspaces` on an OpenAI SDK workload, `projects` on an Anthropic SDK workload) keeps `bedrock-runtime`, records `endpoint_assumption`, discloses it in `rationale` / `capability_gaps[]` and re-runs Clarify. A Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision recorded in the plan and the Present Summary — keep the API on `bedrock-runtime` and drop the requirement (capability gap), or move to `mantle_openai_responses` with the Chat → Responses reshape — never a silent `mantle_openai_chat` / `mantle_messages` row. Same rule in prose in the canonical schema-design-aws-ai.md (+ 2 vendored copies via sync-vendored) and the GCP / Azure shadow schema docs. New fixture fixtures/aws-design-ai/chat-source-web-search-continuity-kept: GPT-5.5 Chat Completions source, `runtime_openai_cris` / `chat_completions`, verbatim `endpoint_assumption`, `capability_gaps[]` naming the dropped `web_search` and the rule-1b decision. fixtures/README.md counts 4 aws-design-ai fixtures and names the new case. Tests (tests/test_runtime_default_review_round3.py): Step 0.8 phrases in both design files (and that rule 2 depends on 1b); the decision sentence in all five schema docs; vendored schema docs byte-identical; the new fixture validates against aws-design-ai.schema.json with the expected row shape; corpus invariant — no `mantle_openai_chat` row with `web_search` / `background_inference`, no `mantle_messages` row with `background_inference` / `projects`. Validation: plugin `pytest tests -q` 1532 passed, 1 failed (known test_model_id_lint::test_real_plugin_passes local .venv noise, pre-existing); skills/llm-to-bedrock/scripts 672 passed; skills/agent-advisor/scripts 570 passed (unchanged); scripts/validate-artifacts.py --fixtures PASS (103 artifacts); tools/sync-vendored.py --check PASS (64); tools/check-skill-refs.py --check PASS; git diff --check clean; markdownlint-cli2 0 issues. --- .../aws-startup-advisor/fixtures/README.md | 2 +- .../aws-design-ai.json | 77 +++++++++++++++++++ .../references/phases/design/design-ai.md | 18 ++++- .../references/shared/schema-design-aws-ai.md | 2 +- .../vendored/ai/schema-design-aws-ai.md | 1 + .../references/phases/design/design-ai.md | 3 +- .../references/shared/schema-design-aws-ai.md | 2 +- .../vendored/ai/schema-design-aws-ai.md | 1 + .../skills/shared/ai/schema-design-aws-ai.md | 1 + .../test_runtime_default_review_round3.py | 74 ++++++++++++++++++ 10 files changed, 176 insertions(+), 5 deletions(-) create mode 100644 plugins/aws-startup-advisor/fixtures/aws-design-ai/chat-source-web-search-continuity-kept/aws-design-ai.json diff --git a/plugins/aws-startup-advisor/fixtures/README.md b/plugins/aws-startup-advisor/fixtures/README.md index bd898f978..d12895734 100644 --- a/plugins/aws-startup-advisor/fixtures/README.md +++ b/plugins/aws-startup-advisor/fixtures/README.md @@ -4,7 +4,7 @@ Also see: - `ai-migration-matrix/` — 14 target-only representative contract snapshots for infrastructure-only, AI-only, and combined migration modes. Under Option A, each run has zero or one infrastructure source; polycloud or multiple simultaneous infrastructure sources are deferred. Any one supported infrastructure source composes independently with many AI sources and optional gateways. `ai_sources[]` retains provider/service provenance, families, and exact normalized `models[]`; `model_evidence` distinguishes observed models from an explicitly empty not-observed set without fake IDs. One OpenRouter gateway can retain multiple structured provider/service/family/exact-model associations, while `upstream_evidence: unresolved` makes unknown upstreams explicit. Synthetic positive tests cover GCP, Azure, and Heroku composition; Heroku with direct Google/Gemini, OpenAI/GPT, and Anthropic/Claude; and one multi-provider, multi-family, multi-model OpenRouter scenario without Cartesian fixture directories. `expected-matrix.json` records represented contract coverage, not a versioned runtime capability catalog: free-form schema validity does not imply an implemented adapter or route. This is contract/test/documentation coverage only and changes no runtime behavior. Combined cases use one `.phase-status.json` and a separate fail-closed `integration-validation.json` cross-track verdict. - `ai-workload-profiles/` — 6 representative `ai-workload-profile.json` observation fixtures (not a Cartesian matrix). Four are schema 1.0: AI-only multi-direct-provider (Anthropic + Google/Vertex, no gateway); OpenAI direct plus Azure OpenAI with distinct `source_service` provenance; a multi-upstream OpenRouter gateway carrying Anthropic (two exact Claude models), Google, and OpenAI simultaneously; and an unresolved/not-yet-identified OpenRouter gateway with an explicit empty upstream set. Two are schema 1.1: `v11-framework-vision` (LangChain with text and vision workloads) and `v11-agentic-tools` (one LangGraph agent with its tool manifest). Each source retains the raw/observed model string in `model_observations[]` alongside the exact normalized `models[]` entry, with a deterministic `provenance`/`confidence` evidence vocabulary. `test_ai_workload_profile_contracts.py` proves every fixture's normalized `ai_sources[]`/`gateways[]` projects exactly into `ai-migration-scenario.schema.json`'s semantics (`test_ai_migration_matrix_contracts.py`), and exercises negative mutations (duplicate identity, a model dropped under a resolved upstream, a fabricated `unknown` ID, OpenRouter named as a direct provider, an empty upstream on an observed gateway, an invented upstream on an unresolved gateway) rather than separate fixture directories per negative. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support — no producer of this artifact exists yet. `ai-workload-profile.schema.json`'s `aiSource` observed branch now also requires `model_observations` non-empty (`minItems: 1`) whenever `model_evidence == "observed"`, matching the identical rule already enforced for `aws-design-ai.schema.json`'s `metadata.ai_sources[]`. -- `aws-design-ai/` — 2 representative target-only `aws-design-ai.json` design fixtures, nested as `/aws-design-ai.json` so `lint:artifacts` discovers them (unlike a flat layout, which #415's own review flagged as invisible to the lint): `multi-provider-direct` (one Anthropic-sourced and one OpenAI-sourced `design_blocks[]` entry in a single design, no gateway) and `openrouter-gateway-multi-upstream` (one OpenRouter gateway fronting Anthropic — two exact Claude models — and OpenAI simultaneously, with a 3-tier routing strategy). `metadata.ai_sources[]`/`metadata.gateways[]` replace the retired single-scalar `metadata.ai_source` field this contract previously carried; `test_aws_design_ai_contracts.py` proves both fixtures project exactly into `ai-workload-profile.schema.json`'s semantics (`test_ai_workload_profile_contracts.py`, #415's own oracle), and exercises negative mutations — the retired single-scalar `ai_source` shape (with and without a `both`-style multi-provider implication), duplicate source identity, OpenRouter named as a direct provider, a dropped model under a resolved gateway upstream, a `design_blocks[]` row without exactly one target, and a fabricated `unknown` model ID. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support, and does not claim either skill's current `design-ai.md` emits this shape — no producer has been repointed to this canonical contract in this change. +- `aws-design-ai/` — 4 representative target-only `aws-design-ai.json` design fixtures, nested as `/aws-design-ai.json` so `lint:artifacts` discovers them (unlike a flat layout, which #415's own review flagged as invisible to the lint): `multi-provider-direct` (one Anthropic-sourced and one OpenAI-sourced `design_blocks[]` entry in a single design, no gateway), `openrouter-gateway-multi-upstream` (one OpenRouter gateway fronting Anthropic — two exact Claude models — and OpenAI simultaneously, with a 3-tier routing strategy), `runtime-default-no-requirement` (a `runtime_openai_cris` row carrying the verbatim `endpoint_assumption` when Clarify recorded no Mantle-only requirement) and `chat-source-web-search-continuity-kept` (a GPT-5.5 Chat Completions source whose confirmed `web_search` requirement needs the Responses API on bedrock-mantle; Step 0.8 rule 1b keeps `chat_completions` on `bedrock-runtime` by explicit decision and discloses the dropped selector in `capability_gaps[]` instead of recording a silent `mantle_openai_chat` row). `metadata.ai_sources[]`/`metadata.gateways[]` replace the retired single-scalar `metadata.ai_source` field this contract previously carried; `test_aws_design_ai_contracts.py` proves both fixtures project exactly into `ai-workload-profile.schema.json`'s semantics (`test_ai_workload_profile_contracts.py`, #415's own oracle), and exercises negative mutations — the retired single-scalar `ai_source` shape (with and without a `both`-style multi-provider implication), duplicate source identity, OpenRouter named as a direct provider, a dropped model under a resolved gateway upstream, a `design_blocks[]` row without exactly one target, and a fabricated `unknown` model ID. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support, and does not claim either skill's current `design-ai.md` emits this shape — no producer has been repointed to this canonical contract in this change. - `estimation-ai/` — 2 representative target-only `estimation-ai.json` cost-estimate fixtures, nested as `/estimation-ai.json` so `lint:artifacts` discovers them: `multi-provider-direct` and `openrouter-gateway-multi-upstream`, reusing #416's `aws-design-ai/` scenario names and the same providers/models. `cost_by_source[]` carries forward `aws-design-ai.json`'s `metadata.ai_sources[]`/`gateways[]` identity pattern so a per-source/per-model estimated cost traces back to the exact Design entry; `model_observations[]` is intentionally not carried forward (the identity is already resolved by Design). `test_estimation_ai_contracts.py` proves both fixtures project into (are a subset of) the matching `aws-design-ai//aws-design-ai.json`, and exercises negative mutations — duplicate/overlapping source identity, OpenRouter named as a direct provider, a fabricated untraceable model, and a model dropped under a resolved gateway upstream. Schema validity here does not imply an implemented pricing lookup, model mapping, or runtime support, and does not claim either skill's current `estimate-ai.md` emits this shape — no producer has been repointed to this canonical contract in this change. - `generation-ai/` — 2 representative target-only `generation-ai.json` migration-plan fixtures, nested as `/generation-ai.json`, mirroring `estimation-ai/`'s two case names. `artifacts_by_source[]` carries the traceability chain one phase forward: a per-source/per-model generated-artifact reference traces back to the exact `estimation-ai.json → cost_by_source[]` entry it was produced from. `test_generation_ai_contracts.py` proves both fixtures project into the matching `estimation-ai//estimation-ai.json`, and exercises the same negative-mutation categories one phase forward. Schema validity here does not imply an implemented adapter, rollout mechanism, or runtime support, and does not claim either skill's current `generate-ai.md`/`generate-artifacts-ai.md` emits this shape. - `azure-rdfa-report*/` — producer-shaped RDfA source fixtures: inventory rows, metrics diff --git a/plugins/aws-startup-advisor/fixtures/aws-design-ai/chat-source-web-search-continuity-kept/aws-design-ai.json b/plugins/aws-startup-advisor/fixtures/aws-design-ai/chat-source-web-search-continuity-kept/aws-design-ai.json new file mode 100644 index 000000000..80baca831 --- /dev/null +++ b/plugins/aws-startup-advisor/fixtures/aws-design-ai/chat-source-web-search-continuity-kept/aws-design-ai.json @@ -0,0 +1,77 @@ +{ + "phase": "design", + "focus": "ai", + "timestamp": "2026-11-09T02:20:00Z", + "source_profile": "ai-workload-profile.json", + "metadata": { + "ai_sources": [ + { + "provider": "openai", + "source_service": "openai_api", + "model_families": ["gpt"], + "model_evidence": "observed", + "models": ["gpt-5.5"], + "model_observations": [ + { + "normalized_model": "gpt-5.5", + "raw_model": "gpt-5.5", + "model_family": "gpt", + "evidence": [{"provenance": "code", "confidence": "high", "detail": "src/ai/openai_client.py:9 client.chat.completions.create"}] + } + ] + } + ], + "gateways": [], + "bedrock_models_selected": ["us.openai.gpt-5.5"], + "regional_validation": "checked" + }, + "design_blocks": [ + { + "workload_id": "wl_openai_research_bot", + "model_id": "gpt-5.5", + "target_bedrock_model": "us.openai.gpt-5.5", + "target_aws_service": null, + "capability": "text_generation", + "capability_confidence": "high", + "rationale": "OpenAI Chat Completions research bot moves to GPT-5.5 on the bedrock-runtime CRIS path. Clarify confirmed a web_search requirement (user), which requires the Responses API on bedrock-mantle; by explicit API-continuity decision (Step 0.8 rule 1b) the Chat Completions API is kept on bedrock-runtime and Web Search is dropped and disclosed as a capability gap rather than recorded as a silent mantle_openai_chat row.", + "confidence_warning": null + } + ], + "ai_architecture": { + "honest_assessment": "strong_migrate", + "honest_assessment_reason": null, + "tiered_strategy": null, + "bedrock_models": [ + { + "source_model_id": "gpt-5.5", + "aws_model_id": "us.openai.gpt-5.5", + "capabilities_matched": ["text_generation"], + "capability_gaps": ["web_search confirmed in Clarify (user) requires the Responses API on bedrock-mantle; the Chat Completions API was kept on bedrock-runtime by explicit decision (Step 0.8 rule 1b) and Web Search is not provided"], + "honest_assessment": "strong_migrate", + "source_provider_price": "2.50/15.00 per 1M tokens", + "bedrock_price": "2.50/15.00 per 1M tokens", + "price_comparison": "parity", + "migration_complexity": "low", + "model_change": false, + "migration_path": "runtime_openai_cris", + "required_client_api": "chat_completions", + "endpoint_assumption": "No Mantle-only requirement was recorded for wl_openai_research_bot; bedrock-runtime selected. If this workload needs background inference, server-side tools, Projects/Workspaces, or single-Region processing, re-run Clarify.", + "quota_risk": "low" + } + ], + "capability_mapping": { + "text_generation": {"parity": "full", "notes": "The runtime OpenAI-compatible Chat Completions API covers the detected usage; Web Search is a server-side tool on bedrock-mantle Responses only and was dropped by explicit decision."} + }, + "code_migration": { + "primary_pattern": "direct_sdk", + "framework": "none", + "files_to_modify": ["src/ai/openai_client.py"], + "dependency_changes": "openai>=2.45.0 with aws_bedrock_token_generator for the OpenAI client; base URL and model id change only." + }, + "infrastructure": [], + "services_to_migrate": [] + }, + "regional_warnings": [], + "multi_model_warnings": [], + "agentic_design": null +} diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md index f0e7352f4..cbe7e883d 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md @@ -112,7 +112,23 @@ Per workload (`workloads[]` from `preferences.json`, else the profile; `*` appli 1. Match `value[]` entries whose `workload_id` is the workload's id or `*`, whose `selector` is one of `background_inference | server_side_tools | web_search | projects | workspaces | in_region_processing`, and whose `source` is `detected` or `user`. `none` matches nothing. -2. **Match → Mantle path for the source API** (`mantle_openai_responses` / `mantle_openai_chat` / +1b. **Validate the matched selector against the API it requires.** `web_search` and + `background_inference` require the Responses API (`web-search.html`; + `inference-responses-api.html` — read 2026-11-09); `workspaces` requires the Anthropic Messages + API (`anthropic-workspace-id`); `projects` requires an OpenAI-compatible API (`OpenAI-Project`). + A selector whose provider does not match the workload's source API — `workspaces` on an OpenAI + SDK workload, `projects` on an Anthropic SDK workload — is a mismatch: the row keeps + `bedrock-runtime`, records `endpoint_assumption`, discloses the mismatch in `rationale` and + `capability_gaps[]`, and Clarify is re-run; never invent a Mantle path. A Chat Completions + source (or an Anthropic Messages source) matched by a Responses-only selector (`web_search`, + `background_inference`) is an **explicit API-continuity decision** recorded in the plan and the + Present Summary, never a silent same-API Mantle row: (i) keep the source API on + `bedrock-runtime` and drop the requirement — the row stays `runtime_openai_cris` / + `runtime_messages` with `endpoint_assumption` and a `capability_gaps[]` entry naming the dropped + selector and the decision; or (ii) move to `mantle_openai_responses` with that + `mantle_selector`, `mantle_requirement`, and the Chat → Responses reshape delta in Generate. + `mantle_openai_chat` / `mantle_messages` is never recorded for a Responses-only selector. +2. **Match that passes 1b → Mantle path for the source API** (`mantle_openai_responses` / `mantle_openai_chat` / `mantle_messages`) with `mantle_selector` = the matched selector (several → first in the order above, name the rest in `rationale`) and `mantle_requirement` copied from the entry: `{ workload_id, selector, source, evidence? (when detected), region? (iff in_region_processing) }`. diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md index dbb5037fb..e8352ef64 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md @@ -177,7 +177,7 @@ web_search | projects | workspaces | in_region_processing` and whose `source` is comes from the model catalog with `source: "catalog"`, never from Clarify. A matching entry on a workload whose source API has no Mantle path (a cross-family `direct` / `converse` / `gpt-oss` target) keeps `bedrock-runtime` and the verbatim `endpoint_assumption`, and discloses the -unhonored selector in `rationale` and `capability_gaps[]`. Every other row keeps +unhonored selector in `rationale` and `capability_gaps[]`. A matched selector is validated against the API it requires before the path is chosen (`web_search` and `background_inference` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. Every other row keeps `bedrock-runtime` and records `endpoint_assumption` — string, REQUIRED on every non-`mantle_*` row, exactly: `No Mantle-only requirement was recorded for ; bedrock-runtime selected. If this workload needs background inference, server-side tools, Projects/Workspaces, or diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md index 60376a2ca..74a9144d5 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -247,6 +247,7 @@ workload: `in_region_processing` applies only when `bedrock-runtime` serves the model in the entry's `region` through a CRIS profile alone (the CRIS-only gate in `ai-openai-to-bedrock.md`); otherwise the workload stays on `bedrock-runtime` in that Region. +- A matched selector is validated against the API it requires before the path is chosen (`web_search` and `background_inference` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. - A matching entry on a workload whose source API has no Mantle path (Gemini / Vertex AI, or a workload whose target is a cross-family `direct` / `converse` / `gpt-oss` row) does not change the endpoint: the row keeps `bedrock-runtime`, records `endpoint_assumption` verbatim (below), diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index 1a8b21419..651520741 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -112,7 +112,8 @@ Read `preferences.json` → `ai_constraints.mantle_requirements` (written by Cla **Derivation rule — apply per workload (`workloads[]` from `preferences.json`, falling back to the profile; when the profile has no `workloads[]`, every entry is `*` and applies to every row):** 1. Collect the `mantle_requirements.value[]` entries whose `workload_id` equals the workload's id or is `*`, and whose `selector` is one of `background_inference | server_side_tools | web_search | projects | workspaces | in_region_processing` with `source` ∈ `detected | user`. Entries with `selector: "none"` or `source: "none"` match nothing. -2. **Any match → the Mantle path for the workload's source API:** `"mantle_openai_responses"` for a Responses source, `"mantle_openai_chat"` for a Chat Completions source, `"mantle_messages"` for an Anthropic SDK source. Set `bedrock_models[].mantle_selector` to the matched selector; when several entries match, take the first in the policy-table order `background_inference, server_side_tools, web_search, projects, workspaces, in_region_processing` and name the others in the row's `rationale`. Copy the matched entry into `bedrock_models[].mantle_requirement` — `{ workload_id, selector, source, evidence?, region? }` where `evidence` is present when `source` is `detected` (the Clarify entry's `file:line construct`) and `region` is present iff `selector` is `in_region_processing`. +1b. **Validate the matched selector against the API it requires.** `web_search` and `background_inference` require the Responses API (`web-search.html`; `inference-responses-api.html` — read 2026-11-09); `workspaces` requires the Anthropic Messages API (`anthropic-workspace-id`); `projects` requires an OpenAI-compatible API (`OpenAI-Project`). A selector whose provider does not match the workload's source API — `workspaces` on an OpenAI SDK workload, `projects` on an Anthropic SDK workload — is a mismatch: the row keeps `bedrock-runtime`, records `endpoint_assumption`, discloses the mismatch in `rationale` and `capability_gaps[]`, and Clarify is re-run; never invent a Mantle path. A Chat Completions source (or an Anthropic Messages source) matched by a Responses-only selector (`web_search`, `background_inference`) is an **explicit API-continuity decision** recorded in the plan and the Present Summary, never a silent same-API Mantle row: (i) keep the source API on `bedrock-runtime` and drop the requirement — the row stays `runtime_openai_cris` / `runtime_messages` with `endpoint_assumption` and a `capability_gaps[]` entry naming the dropped selector and the decision; or (ii) move to `mantle_openai_responses` with that `mantle_selector`, `mantle_requirement`, and the Chat → Responses reshape delta in Generate. `mantle_openai_chat` / `mantle_messages` is never recorded for a Responses-only selector. +2. **Any match that passes 1b → the Mantle path for the workload's source API:** `"mantle_openai_responses"` for a Responses source, `"mantle_openai_chat"` for a Chat Completions source, `"mantle_messages"` for an Anthropic SDK source. Set `bedrock_models[].mantle_selector` to the matched selector; when several entries match, take the first in the policy-table order `background_inference, server_side_tools, web_search, projects, workspaces, in_region_processing` and name the others in the row's `rationale`. Copy the matched entry into `bedrock_models[].mantle_requirement` — `{ workload_id, selector, source, evidence?, region? }` where `evidence` is present when `source` is `detected` (the Clarify entry's `file:line construct`) and `region` is present iff `selector` is `in_region_processing`. - `in_region_processing` still obeys the existing CRIS-only gate in `references/vendored/ai/ai-openai-to-bedrock.md`: it selects Mantle only when `bedrock-runtime` serves the selected model in the entry's `region` through a CRIS profile alone; when a runtime in-Region form exists, the workload stays on `bedrock-runtime` in that Region and the row records the `endpoint_assumption` below with a `rationale` note that the Region need is met on runtime. - **A match on a workload whose source API has no Mantle path** (Gemini / Vertex AI via `google-genai`, or any workload whose target is a cross-family `"direct"` / `"converse"` / `"gpt-oss"` row) does not change the endpoint. The row keeps its `bedrock-runtime` path and records the `endpoint_assumption` sentence below verbatim (it is the fixed runtime-lane disclosure), and the recorded requirement is disclosed as a capability gap: name the unhonored selector with its `source` and `evidence` in the row's `rationale` and add a `capability_gaps[]` entry (`" requested in Clarify (, ) — no bedrock-mantle path for this source API"`); for `in_region_processing` add a `regional_warnings[]` entry instead. Never invent a Mantle path for that source and never drop the answer silently; the Present Summary endpoint line names the gap. 3. **`mantle_only_model_or_api` is never a Clarify value.** When the selected model or API exists only on `bedrock-mantle` per the catalog (`references/vendored/ai/ai-openai-to-bedrock.md` / `references/shared/openai-on-bedrock.md` access-path tables), record the Mantle path with `mantle_selector: "mantle_only_model_or_api"` and `mantle_requirement: { "workload_id": "", "selector": "mantle_only_model_or_api", "source": "catalog", "evidence": "" }`. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md index 55b811fde..a475aac93 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md @@ -94,7 +94,7 @@ same vocabulary as the shared contract (`references/vendored/ai/schema-design-aw - `mantle_openai_responses` / `mantle_openai_chat` / `mantle_messages` — the `bedrock-mantle` endpoint; each such row MUST carry `mantle_selector`, one of `background_inference | server_side_tools | web_search | projects | workspaces | mantle_only_model_or_api | - in_region_processing`. API continuity alone is not a selector. + in_region_processing`. API continuity alone is not a selector. A matched selector is validated against the API it requires before the path is chosen (`web_search` and `background_inference` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. - `bedrock_models[].mantle_requirement` — object, REQUIRED on every `mantle_*` row, absent otherwise: `{ workload_id, selector, source, evidence?, region? }` — `selector` equals the row's `mantle_selector`; `source` ∈ `detected | user | catalog` (`detected`/`user` entries are copied diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md index 60376a2ca..74a9144d5 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -247,6 +247,7 @@ workload: `in_region_processing` applies only when `bedrock-runtime` serves the model in the entry's `region` through a CRIS profile alone (the CRIS-only gate in `ai-openai-to-bedrock.md`); otherwise the workload stays on `bedrock-runtime` in that Region. +- A matched selector is validated against the API it requires before the path is chosen (`web_search` and `background_inference` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. - A matching entry on a workload whose source API has no Mantle path (Gemini / Vertex AI, or a workload whose target is a cross-family `direct` / `converse` / `gpt-oss` row) does not change the endpoint: the row keeps `bedrock-runtime`, records `endpoint_assumption` verbatim (below), diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md index 60376a2ca..74a9144d5 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md @@ -247,6 +247,7 @@ workload: `in_region_processing` applies only when `bedrock-runtime` serves the model in the entry's `region` through a CRIS profile alone (the CRIS-only gate in `ai-openai-to-bedrock.md`); otherwise the workload stays on `bedrock-runtime` in that Region. +- A matched selector is validated against the API it requires before the path is chosen (`web_search` and `background_inference` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. - A matching entry on a workload whose source API has no Mantle path (Gemini / Vertex AI, or a workload whose target is a cross-family `direct` / `converse` / `gpt-oss` row) does not change the endpoint: the row keeps `bedrock-runtime`, records `endpoint_assumption` verbatim (below), diff --git a/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py index 92a999ab3..2a03bf4a2 100644 --- a/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py +++ b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py @@ -224,3 +224,77 @@ def test_code_rewriter_carries_the_per_model_guardrails_and_caching_scope() -> N assert "cards read 2026-11-09" in text +# --------------------------------------------------------------------------- F4 (cloud half): selector ↔ API validation + +STEP_08_PHRASES = [ + "Validate the matched selector against the API it requires", + "require the Responses API", + "`workspaces` requires the Anthropic Messages API", + "`projects` requires an OpenAI-compatible API", + "explicit API-continuity decision", + "never a silent same-API Mantle row", + "`mantle_openai_chat` / `mantle_messages` is never recorded for a Responses-only selector", + "read 2026-11-09", +] + + +@pytest.mark.parametrize(("path", "end"), [(GCP_DESIGN_AI, "## Part 1"), (AZURE_DESIGN_AI, "Exactly one of")], + ids=["gcp", "azure"]) +def test_step_08_validates_the_selector_against_its_api_before_choosing_a_path(path: Path, end: str) -> None: + step = _squash(_section(path.read_text(), "## Step 0.8", end)) + for phrase in STEP_08_PHRASES: + assert phrase in step, f"{path}: {phrase!r} missing from Step 0.8" + assert "1b." in step + assert re.search(r"2\. \*\*(Any m|M)atch that passes 1b", step), f"{path}: rule 2 must depend on 1b" + assert "keep the source API on `bedrock-runtime` and drop the requirement" in step + assert "move to `mantle_openai_responses`" in step + assert "`workspaces` on an OpenAI SDK workload, `projects` on an Anthropic SDK workload" in step + + +@pytest.mark.parametrize("path", ALL_SCHEMA_DOCS, ids=lambda p: str(p.relative_to(SKILLS))) +def test_schema_docs_carry_the_responses_only_selector_decision(path: Path) -> None: + text = _squash(path.read_text()) + assert "A matched selector is validated against the API it requires before the path is chosen" in text + assert "`web_search` and `background_inference` require the Responses API" in text + assert "explicit API-continuity decision" in text and "never a silent same-API Mantle row" in text + assert "`workspaces` on OpenAI, `projects` on Anthropic" in text + + +def test_vendored_schema_docs_are_byte_identical_to_the_canonical() -> None: + for vendored in VENDORED_SCHEMA_DOCS: + assert vendored.read_bytes() == CANONICAL_SCHEMA_DOC.read_bytes(), vendored + + +def _design_ai_findings(design: dict) -> list: + validator = _load_module("round3_artifact_validator", ARTIFACT_VALIDATOR) + schema = json.loads(DESIGN_AI_SCHEMA.read_text()) + findings: list = [] + validator.validate_json_schema(schema, design, "", "design", str(DESIGN_AI_SCHEMA), findings) + return findings + + +def test_chat_source_web_search_fixture_keeps_the_api_and_discloses_the_dropped_selector() -> None: + design = json.loads(NEW_FIXTURE.read_text()) + assert _design_ai_findings(design) == [] + rows = design["ai_architecture"]["bedrock_models"] + assert len(rows) == 1 + row = rows[0] + assert row["migration_path"] == "runtime_openai_cris" + assert row["required_client_api"] == "chat_completions" + assert "mantle_selector" not in row and "mantle_requirement" not in row + assert row["endpoint_assumption"].startswith("No Mantle-only requirement was recorded for wl_openai_research_bot; bedrock-runtime selected.") + assert any("web_search" in gap and "Responses API" in gap and "explicit decision" in gap for gap in row["capability_gaps"]) + assert "Step 0.8 rule 1b" in design["design_blocks"][0]["rationale"] + assert design["metadata"]["ai_sources"][0]["models"] == ["gpt-5.5"] + assert design["metadata"]["bedrock_models_selected"] == [row["aws_model_id"]] == ["us.openai.gpt-5.5"] + + +def test_design_fixture_corpus_never_records_a_responses_only_selector_on_a_same_api_mantle_row() -> None: + assert NEW_FIXTURE in DESIGN_FIXTURES + for path in DESIGN_FIXTURES: + for row in json.loads(path.read_text())["ai_architecture"]["bedrock_models"]: + selector = row.get("mantle_selector") + if row.get("migration_path") == "mantle_openai_chat": + assert selector not in {"web_search", "background_inference"}, f"{path}: Responses-only selector on a Chat Mantle row" + if row.get("migration_path") == "mantle_messages": + assert selector not in {"background_inference", "projects"}, f"{path}: OpenAI-only selector on a Messages Mantle row" From 4f861123b12314d06196652088879e51cc99bff9 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 9 Nov 2026 09:28:58 -0800 Subject: [PATCH 26/41] fix(aws-startup-advisor): price the web_search continuity fixture on GPT-5.6 Terra, whose Geo CRIS rate is published (review A, B1) The chat-source-web-search-continuity-kept design fixture quoted a parity "2.50/15.00" Bedrock price for us.openai.gpt-5.5, but the GPT-5.5 card publishes a commercial In-Region price only and no Geo/Global CRIS row (model-card-openai-gpt-55.html, read 2026-11-09); bedrock-endpoint-policy.md and openai-on-bedrock.md say such rows must never report parity. Switch the fixture to a same-model GPT-5.6 Terra move: source gpt-5.6-terra at OpenAI's $2.00/$12.00 standard list, target us.openai.gpt-5.6-terra at the published US Geo CRIS $2.20/$13.20 (model-card-openai-gpt-56-terra.html, read 2026-11-09; Global CRIS $2.00/$12.00), price_comparison "unfavorable", with the 1.10x Geo premium named in the rationale. The Step 0.8 rule 1b continuity decision, dropped web_search capability gap, chat_completions API and verbatim endpoint_assumption are unchanged. Round-3 test now pins the Terra ids and prices and asserts price_comparison is not parity; fixtures/README.md names the Terra source. Validation (plugins/aws-startup-advisor): - uv run --no-project --with pytest --with jsonschema python -m pytest tests -q: 1532 passed, 1 failed (known tests/test_model_id_lint.py::test_real_plugin_passes, local .venv noise) - python3 scripts/validate-artifacts.py --fixtures: PASS (103 artifacts, 0 errors) - python3 tools/sync-vendored.py --check: PASS (64 files) - git diff --check: clean --- .../aws-startup-advisor/fixtures/README.md | 2 +- .../aws-design-ai.json | 24 +++++++++---------- .../test_runtime_default_review_round3.py | 9 +++++-- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/plugins/aws-startup-advisor/fixtures/README.md b/plugins/aws-startup-advisor/fixtures/README.md index d12895734..d2d83bb8a 100644 --- a/plugins/aws-startup-advisor/fixtures/README.md +++ b/plugins/aws-startup-advisor/fixtures/README.md @@ -4,7 +4,7 @@ Also see: - `ai-migration-matrix/` — 14 target-only representative contract snapshots for infrastructure-only, AI-only, and combined migration modes. Under Option A, each run has zero or one infrastructure source; polycloud or multiple simultaneous infrastructure sources are deferred. Any one supported infrastructure source composes independently with many AI sources and optional gateways. `ai_sources[]` retains provider/service provenance, families, and exact normalized `models[]`; `model_evidence` distinguishes observed models from an explicitly empty not-observed set without fake IDs. One OpenRouter gateway can retain multiple structured provider/service/family/exact-model associations, while `upstream_evidence: unresolved` makes unknown upstreams explicit. Synthetic positive tests cover GCP, Azure, and Heroku composition; Heroku with direct Google/Gemini, OpenAI/GPT, and Anthropic/Claude; and one multi-provider, multi-family, multi-model OpenRouter scenario without Cartesian fixture directories. `expected-matrix.json` records represented contract coverage, not a versioned runtime capability catalog: free-form schema validity does not imply an implemented adapter or route. This is contract/test/documentation coverage only and changes no runtime behavior. Combined cases use one `.phase-status.json` and a separate fail-closed `integration-validation.json` cross-track verdict. - `ai-workload-profiles/` — 6 representative `ai-workload-profile.json` observation fixtures (not a Cartesian matrix). Four are schema 1.0: AI-only multi-direct-provider (Anthropic + Google/Vertex, no gateway); OpenAI direct plus Azure OpenAI with distinct `source_service` provenance; a multi-upstream OpenRouter gateway carrying Anthropic (two exact Claude models), Google, and OpenAI simultaneously; and an unresolved/not-yet-identified OpenRouter gateway with an explicit empty upstream set. Two are schema 1.1: `v11-framework-vision` (LangChain with text and vision workloads) and `v11-agentic-tools` (one LangGraph agent with its tool manifest). Each source retains the raw/observed model string in `model_observations[]` alongside the exact normalized `models[]` entry, with a deterministic `provenance`/`confidence` evidence vocabulary. `test_ai_workload_profile_contracts.py` proves every fixture's normalized `ai_sources[]`/`gateways[]` projects exactly into `ai-migration-scenario.schema.json`'s semantics (`test_ai_migration_matrix_contracts.py`), and exercises negative mutations (duplicate identity, a model dropped under a resolved upstream, a fabricated `unknown` ID, OpenRouter named as a direct provider, an empty upstream on an observed gateway, an invented upstream on an unresolved gateway) rather than separate fixture directories per negative. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support — no producer of this artifact exists yet. `ai-workload-profile.schema.json`'s `aiSource` observed branch now also requires `model_observations` non-empty (`minItems: 1`) whenever `model_evidence == "observed"`, matching the identical rule already enforced for `aws-design-ai.schema.json`'s `metadata.ai_sources[]`. -- `aws-design-ai/` — 4 representative target-only `aws-design-ai.json` design fixtures, nested as `/aws-design-ai.json` so `lint:artifacts` discovers them (unlike a flat layout, which #415's own review flagged as invisible to the lint): `multi-provider-direct` (one Anthropic-sourced and one OpenAI-sourced `design_blocks[]` entry in a single design, no gateway), `openrouter-gateway-multi-upstream` (one OpenRouter gateway fronting Anthropic — two exact Claude models — and OpenAI simultaneously, with a 3-tier routing strategy), `runtime-default-no-requirement` (a `runtime_openai_cris` row carrying the verbatim `endpoint_assumption` when Clarify recorded no Mantle-only requirement) and `chat-source-web-search-continuity-kept` (a GPT-5.5 Chat Completions source whose confirmed `web_search` requirement needs the Responses API on bedrock-mantle; Step 0.8 rule 1b keeps `chat_completions` on `bedrock-runtime` by explicit decision and discloses the dropped selector in `capability_gaps[]` instead of recording a silent `mantle_openai_chat` row). `metadata.ai_sources[]`/`metadata.gateways[]` replace the retired single-scalar `metadata.ai_source` field this contract previously carried; `test_aws_design_ai_contracts.py` proves both fixtures project exactly into `ai-workload-profile.schema.json`'s semantics (`test_ai_workload_profile_contracts.py`, #415's own oracle), and exercises negative mutations — the retired single-scalar `ai_source` shape (with and without a `both`-style multi-provider implication), duplicate source identity, OpenRouter named as a direct provider, a dropped model under a resolved gateway upstream, a `design_blocks[]` row without exactly one target, and a fabricated `unknown` model ID. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support, and does not claim either skill's current `design-ai.md` emits this shape — no producer has been repointed to this canonical contract in this change. +- `aws-design-ai/` — 4 representative target-only `aws-design-ai.json` design fixtures, nested as `/aws-design-ai.json` so `lint:artifacts` discovers them (unlike a flat layout, which #415's own review flagged as invisible to the lint): `multi-provider-direct` (one Anthropic-sourced and one OpenAI-sourced `design_blocks[]` entry in a single design, no gateway), `openrouter-gateway-multi-upstream` (one OpenRouter gateway fronting Anthropic — two exact Claude models — and OpenAI simultaneously, with a 3-tier routing strategy), `runtime-default-no-requirement` (a `runtime_openai_cris` row carrying the verbatim `endpoint_assumption` when Clarify recorded no Mantle-only requirement) and `chat-source-web-search-continuity-kept` (a GPT-5.6 Terra Chat Completions source whose confirmed `web_search` requirement needs the Responses API on bedrock-mantle; Step 0.8 rule 1b keeps `chat_completions` on `bedrock-runtime` by explicit decision and discloses the dropped selector in `capability_gaps[]` instead of recording a silent `mantle_openai_chat` row). `metadata.ai_sources[]`/`metadata.gateways[]` replace the retired single-scalar `metadata.ai_source` field this contract previously carried; `test_aws_design_ai_contracts.py` proves both fixtures project exactly into `ai-workload-profile.schema.json`'s semantics (`test_ai_workload_profile_contracts.py`, #415's own oracle), and exercises negative mutations — the retired single-scalar `ai_source` shape (with and without a `both`-style multi-provider implication), duplicate source identity, OpenRouter named as a direct provider, a dropped model under a resolved gateway upstream, a `design_blocks[]` row without exactly one target, and a fabricated `unknown` model ID. Schema validity here does not imply an implemented detector, adapter, route, model mapping, or runtime support, and does not claim either skill's current `design-ai.md` emits this shape — no producer has been repointed to this canonical contract in this change. - `estimation-ai/` — 2 representative target-only `estimation-ai.json` cost-estimate fixtures, nested as `/estimation-ai.json` so `lint:artifacts` discovers them: `multi-provider-direct` and `openrouter-gateway-multi-upstream`, reusing #416's `aws-design-ai/` scenario names and the same providers/models. `cost_by_source[]` carries forward `aws-design-ai.json`'s `metadata.ai_sources[]`/`gateways[]` identity pattern so a per-source/per-model estimated cost traces back to the exact Design entry; `model_observations[]` is intentionally not carried forward (the identity is already resolved by Design). `test_estimation_ai_contracts.py` proves both fixtures project into (are a subset of) the matching `aws-design-ai//aws-design-ai.json`, and exercises negative mutations — duplicate/overlapping source identity, OpenRouter named as a direct provider, a fabricated untraceable model, and a model dropped under a resolved gateway upstream. Schema validity here does not imply an implemented pricing lookup, model mapping, or runtime support, and does not claim either skill's current `estimate-ai.md` emits this shape — no producer has been repointed to this canonical contract in this change. - `generation-ai/` — 2 representative target-only `generation-ai.json` migration-plan fixtures, nested as `/generation-ai.json`, mirroring `estimation-ai/`'s two case names. `artifacts_by_source[]` carries the traceability chain one phase forward: a per-source/per-model generated-artifact reference traces back to the exact `estimation-ai.json → cost_by_source[]` entry it was produced from. `test_generation_ai_contracts.py` proves both fixtures project into the matching `estimation-ai//estimation-ai.json`, and exercises the same negative-mutation categories one phase forward. Schema validity here does not imply an implemented adapter, rollout mechanism, or runtime support, and does not claim either skill's current `generate-ai.md`/`generate-artifacts-ai.md` emits this shape. - `azure-rdfa-report*/` — producer-shaped RDfA source fixtures: inventory rows, metrics diff --git a/plugins/aws-startup-advisor/fixtures/aws-design-ai/chat-source-web-search-continuity-kept/aws-design-ai.json b/plugins/aws-startup-advisor/fixtures/aws-design-ai/chat-source-web-search-continuity-kept/aws-design-ai.json index 80baca831..bf805a18e 100644 --- a/plugins/aws-startup-advisor/fixtures/aws-design-ai/chat-source-web-search-continuity-kept/aws-design-ai.json +++ b/plugins/aws-startup-advisor/fixtures/aws-design-ai/chat-source-web-search-continuity-kept/aws-design-ai.json @@ -10,11 +10,11 @@ "source_service": "openai_api", "model_families": ["gpt"], "model_evidence": "observed", - "models": ["gpt-5.5"], + "models": ["gpt-5.6-terra"], "model_observations": [ { - "normalized_model": "gpt-5.5", - "raw_model": "gpt-5.5", + "normalized_model": "gpt-5.6-terra", + "raw_model": "gpt-5.6-terra", "model_family": "gpt", "evidence": [{"provenance": "code", "confidence": "high", "detail": "src/ai/openai_client.py:9 client.chat.completions.create"}] } @@ -22,18 +22,18 @@ } ], "gateways": [], - "bedrock_models_selected": ["us.openai.gpt-5.5"], + "bedrock_models_selected": ["us.openai.gpt-5.6-terra"], "regional_validation": "checked" }, "design_blocks": [ { "workload_id": "wl_openai_research_bot", - "model_id": "gpt-5.5", - "target_bedrock_model": "us.openai.gpt-5.5", + "model_id": "gpt-5.6-terra", + "target_bedrock_model": "us.openai.gpt-5.6-terra", "target_aws_service": null, "capability": "text_generation", "capability_confidence": "high", - "rationale": "OpenAI Chat Completions research bot moves to GPT-5.5 on the bedrock-runtime CRIS path. Clarify confirmed a web_search requirement (user), which requires the Responses API on bedrock-mantle; by explicit API-continuity decision (Step 0.8 rule 1b) the Chat Completions API is kept on bedrock-runtime and Web Search is dropped and disclosed as a capability gap rather than recorded as a silent mantle_openai_chat row.", + "rationale": "OpenAI Chat Completions research bot stays on GPT-5.6 Terra via the bedrock-runtime US Geo CRIS id (same model, no model change); the US Geo CRIS rate is 1.10x OpenAI's $2.00/$12.00 standard list price (model-card-openai-gpt-56-terra.html, read 2026-11-09), so the comparison is unfavorable in-Geo and parity only on Global CRIS. Clarify confirmed a web_search requirement (user), which requires the Responses API on bedrock-mantle; by explicit API-continuity decision (Step 0.8 rule 1b) the Chat Completions API is kept on bedrock-runtime and Web Search is dropped and disclosed as a capability gap rather than recorded as a silent mantle_openai_chat row.", "confidence_warning": null } ], @@ -43,14 +43,14 @@ "tiered_strategy": null, "bedrock_models": [ { - "source_model_id": "gpt-5.5", - "aws_model_id": "us.openai.gpt-5.5", + "source_model_id": "gpt-5.6-terra", + "aws_model_id": "us.openai.gpt-5.6-terra", "capabilities_matched": ["text_generation"], "capability_gaps": ["web_search confirmed in Clarify (user) requires the Responses API on bedrock-mantle; the Chat Completions API was kept on bedrock-runtime by explicit decision (Step 0.8 rule 1b) and Web Search is not provided"], "honest_assessment": "strong_migrate", - "source_provider_price": "2.50/15.00 per 1M tokens", - "bedrock_price": "2.50/15.00 per 1M tokens", - "price_comparison": "parity", + "source_provider_price": "2.00/12.00 per 1M tokens", + "bedrock_price": "2.20/13.20 per 1M tokens", + "price_comparison": "unfavorable", "migration_complexity": "low", "model_change": false, "migration_path": "runtime_openai_cris", diff --git a/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py index 2a03bf4a2..05ad3aeb2 100644 --- a/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py +++ b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py @@ -285,8 +285,13 @@ def test_chat_source_web_search_fixture_keeps_the_api_and_discloses_the_dropped_ assert row["endpoint_assumption"].startswith("No Mantle-only requirement was recorded for wl_openai_research_bot; bedrock-runtime selected.") assert any("web_search" in gap and "Responses API" in gap and "explicit decision" in gap for gap in row["capability_gaps"]) assert "Step 0.8 rule 1b" in design["design_blocks"][0]["rationale"] - assert design["metadata"]["ai_sources"][0]["models"] == ["gpt-5.5"] - assert design["metadata"]["bedrock_models_selected"] == [row["aws_model_id"]] == ["us.openai.gpt-5.5"] + assert design["metadata"]["ai_sources"][0]["models"] == ["gpt-5.6-terra"] + assert design["metadata"]["bedrock_models_selected"] == [row["aws_model_id"]] == ["us.openai.gpt-5.6-terra"] + # The Geo CRIS rate is published on the GPT-5.6 Terra card; GPT-5.5 / GPT-5.4 publish no Geo/Global + # price, so a runtime CRIS row for them must never carry a verified-looking parity price. + assert row["bedrock_price"] == "2.20/13.20 per 1M tokens" + assert row["source_provider_price"] == "2.00/12.00 per 1M tokens" + assert row["price_comparison"] != "parity" def test_design_fixture_corpus_never_records_a_responses_only_selector_on_a_same_api_mantle_row() -> None: From 3cf0a9a57fad33ba65f26cf65aebf960a5dfe4e8 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 7 Nov 2026 21:35:26 -0800 Subject: [PATCH 27/41] feat(aws-startup-advisor): add bedrock-runtime Responses/Chat/Messages paths to the agent-advisor catalogs, schemas and verifier PR 3 of 3 (agent-advisor bedrock-runtime default), FEAT-001: path vocabulary only. The recommendation engines are not changed in this commit; they ignore unknown catalog path keys and the new schema fields are optional. Catalogs (canonical skills/shared/ai/models, vendored copies re-synced): - openai-bedrock.json: new paths.runtime_openai_responses and paths.runtime_openai_chat (OpenAI SDK against https://bedrock-runtime.{region}.amazonaws.com/openai/v1, bedrock:InvokeModel, cris_inference_profile) plus rows on all nine openai_gpt_* models (available per each card's bedrock-runtime Responses / Chat Completions column, requires_cris true, bare model_id; Astra carries supported_regions, inference_profile_regions and path capabilities). runtime_converse strengths[1] now says Guardrails and invocation logging are bedrock-runtime-only while CloudWatch metrics exist on both endpoints (AWS/Bedrock, AWS/BedrockMantle; monitoring-mantle-metrics.html). No existing row's available flag or evidence changed. - anthropic-bedrock.json: new paths.runtime_messages (Anthropic SDK against https://bedrock-runtime.{region}.amazonaws.com/anthropic) and rows on claude_opus_5_5, claude_sonnet_5, claude_haiku_4_5 using each model's runtime_converse model_id. - Both top-level `source` strings name the new rows and the read date. Schemas (skills/agent-advisor/scripts/schemas): - model-recommendation-input.json: preferred_api_path += runtime_messages, runtime_openai_chat, runtime_openai_responses; data_residency += in_region_required; new requirements.mantle_selector (7-value enum from skills/shared/ai/bedrock-endpoint-policy.md). - model-recommendation.json: apiPath += the 3 paths; optional mantle_selector (enum + null) on recommendation and candidate. - model-verification.json: apiPath += the 3 paths. Verifier (verify_model_path.py): _default_openai_client takes the base URL; new _default_runtime_openai_responses_client / _default_runtime_openai_chat_client (bedrock-runtime /openai/v1) and _default_runtime_messages_client (anthropic.Anthropic at bedrock-runtime /anthropic with a Bedrock bearer token); runtime_openai_responses_client_factory, runtime_openai_chat_client_factory and runtime_messages_client_factory kwargs on verify_workload / verify_recommendation; dispatch for the three paths. Application-profile pre-check and the not-implemented fallback unchanged. Vendoring: skills/agent-advisor/references/vendored/ai/bedrock-endpoint-policy.md (byte-identical copy, README row and prose); tests/test_endpoint_policy_contract.py checks the agent-advisor copy. Tests: new scripts/test_catalog_paths.py (catalog/schema path-enum agreement, row shape, CRIS invariants, evidence citations, A6 wording); 15 additions to test_verify_model_path.py (exact CRIS id reaches each Runtime fake, structured failure without substitution, schema validation, every apiPath enum value probed, default client builders name the missing package and point at the right base URL). Model cards read 2026-11-07 for every new row (all under https://docs.aws.amazon.com/bedrock/latest/userguide/): model-card-openai-gpt-56-sol.html, model-card-openai-gpt-56-terra.html, model-card-openai-gpt-56-luna.html, model-card-openai-gpt-55.html, model-card-openai-gpt-54.html, model-card-openai-gpt-6-astra.html, model-card-openai-gpt-6-sol.html, model-card-openai-gpt-6-luna.html, model-card-openai-gpt-6-1-sol.html, model-card-anthropic-claude-opus-5-5.html, model-card-anthropic-claude-sonnet-5.html, model-card-anthropic-claude-haiku-4-5.html. Policy pages read 2026-11-07: endpoints.html, inference-responses-api.html, inference-chat-completions.html, inference-messages-api.html, workspaces.html, monitoring-mantle-metrics.html, model-invocation-logging.html. Upstream alignment: the new path definitions and model-path rows use only the field names PR #567's scripts/contracts/model-selection-catalog.schema.json allows (pathDefinition: sdk, client, endpoint, api, iam_action, iam_auth, model_id_strategy, supports_cris, strengths, constraints; modelPath: available, model_id, requires_cris, evidence, capabilities, supported_regions, inference_profile_regions). No selection_status field and no Sonnet 5.5 / Haiku 5.5 / GPT-6.1 Terra rows are added (issue #566 stage 2). Verification (worktree P = plugins/aws-startup-advisor): - cd P/skills/agent-advisor/scripts && uv run --group dev pytest -q -> 602 passed (570 baseline + 32 new) - cd P && python3 tools/sync-vendored.py --check -> PASS, 65 vendored files byte-identical (64 at baseline) - cd P && uv run --no-project --with pytest python -m pytest tests/test_gpt54_55_cris_prefix_scope.py tests/test_endpoint_policy_contract.py -q -> 11 passed - mise run lint:skill-refs -> PASS, 1524 references, 0 missing, 7 baselined - mise run lint:model-catalog-drift -> OK, 3 pre-existing warnings (claude-sonnet-5-5, claude-haiku-5-5, gpt-6.1-terra absent by design) - mise run lint:model-id -> fails only on the untracked, git-ignored local P/skills/llm-to-bedrock/scripts/.venv/ (anthropic/_constants.py names a retired Opus 4.1 id); the same lint on a copy of the tracked tree without that .venv -> OK (19 retired IDs enforced) - mise run lint:startup-model-pricing -> OK - markdownlint-cli2 on the edited vendored README and the new vendored policy copy -> 0 errors - git diff --check -> clean - gitleaks (root `mise run build`) not run: not installed locally; the known pre-existing upstream finding on f8d09fd1 is unrelated to this commit. --- .../references/vendored/README.md | 8 +- .../vendored/ai/bedrock-endpoint-policy.md | 115 ++++++++ .../vendored/ai/models/anthropic-bedrock.json | 38 ++- .../vendored/ai/models/openai-bedrock.json | 243 ++++++++++++++++- .../schemas/model-recommendation-input.json | 18 +- .../scripts/schemas/model-recommendation.json | 23 +- .../scripts/schemas/model-verification.json | 5 +- .../scripts/test_catalog_paths.py | 128 +++++++++ .../scripts/test_verify_model_path.py | 248 ++++++++++++++++++ .../scripts/verify_model_path.py | 77 +++++- .../shared/ai/models/anthropic-bedrock.json | 38 ++- .../shared/ai/models/openai-bedrock.json | 243 ++++++++++++++++- .../tests/test_endpoint_policy_contract.py | 5 +- 13 files changed, 1161 insertions(+), 28 deletions(-) create mode 100644 plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-endpoint-policy.md create mode 100644 plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_catalog_paths.py diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md index 323d25535..a4bd26550 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/README.md @@ -15,14 +15,16 @@ see a new `skills/shared/` file no skill vendors yet — adding one here means t | Vendored path | Canonical source | | ---------------------------------- | ------------------------------------------------ | +| `ai/bedrock-endpoint-policy.md` | `skills/shared/ai/bedrock-endpoint-policy.md` | | `ai/bedrock-pricing-cache.md` | `skills/shared/ai/bedrock-pricing-cache.md` | | `ai/models/anthropic-bedrock.json` | `skills/shared/ai/models/anthropic-bedrock.json` | | `ai/models/openai-bedrock.json` | `skills/shared/ai/models/openai-bedrock.json` | | `dsl/INTERPRETER.md` | `skills/shared/dsl/INTERPRETER.md` | -This skill vendors the DSL interpreter contract, the provider-neutral Bedrock pricing -cache, and the Anthropic/OpenAI model-selection catalogs read by -`scripts/model_recommendation.py`. It does not vendor the shared state schema +This skill vendors the DSL interpreter contract, the Bedrock endpoint policy +(`bedrock-runtime` default, `mantle_selector` enum) obeyed by Model Recommend, the +provider-neutral Bedrock pricing cache, and the Anthropic/OpenAI model-selection catalogs +read by `scripts/model_recommendation.py`. It does not vendor the shared state schema (`skills/shared/state/phase-status.schema.json`) — agent-advisor's `.phase-status.json` carries advisor-specific keys and statuses, declared in SKILL.md § State file per INTERPRETER.md § Skill bindings. diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-endpoint-policy.md new file mode 100644 index 000000000..fdc4948a2 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-endpoint-policy.md @@ -0,0 +1,115 @@ +# Bedrock Endpoint Policy — `bedrock-runtime` by default, `bedrock-mantle` by selector + +> Canonical endpoint-selection policy for every Bedrock migration path in this plugin. +> Vendored into each consuming skill as `references/vendored/ai/bedrock-endpoint-policy.md` +> and kept byte-identical by `tools/sync-vendored.py`; edit HERE, then run the sync. +> Every statement below comes from the public Amazon Bedrock User Guide pages listed in +> § Sources (read 2026-11-07). Re-read those pages before changing a row. + +## The rule + +**Use `bedrock-runtime` for new applications.** Amazon Bedrock documents `bedrock-runtime` as the +recommended endpoint for most new applications; it serves InvokeModel, Converse, the +OpenAI-compatible Responses and Chat Completions APIs, and the Anthropic Messages API. + +**Use `bedrock-mantle` only for a verified Mantle-only need.** The documented reasons are: + +- background (asynchronous, long-running) inference — Responses `background=true` is rejected + with a 400 on `bedrock-runtime`; +- server-side tool use and pre-configured tools, including Web Search; +- customer-created Projects (OpenAI-compatible) or Workspaces (Anthropic-compatible) — + `bedrock-runtime` exposes only the account's default project; +- a model or API that the Endpoint availability page lists on `bedrock-mantle` only; +- strict in-Region processing for a model that `bedrock-runtime` serves only through a + cross-Region inference (CRIS) profile while `bedrock-mantle` serves it in the required Region. + +**API continuity alone never selects Mantle.** An OpenAI Responses, OpenAI Chat Completions or +Anthropic Messages source keeps its SDK on `bedrock-runtime` by changing the base URL, the +credential and the model ID. Choosing Mantle because "the app already speaks Responses" is not a +selector. + +**Pricing never selects an endpoint.** Per-token pricing for the same model is identical on both +endpoints; choose by API and capability need. + +## `mantle_selector` — the recorded reason + +Every `bedrock_models[]` row whose `migration_path` starts with `mantle_` (`mantle_openai_responses`, +`mantle_openai_chat`, `mantle_messages`) MUST record `mantle_selector` with exactly one value: + +| `mantle_selector` | Verified need | +| --------------------------- | ----------------------------------------------------------------------------------------------- | +| `background_inference` | Responses requests with `background=true` / long-running asynchronous inference | +| `server_side_tools` | Server-side tool use (Lambda-backed MCP tools) or pre-configured tools other than Web Search | +| `web_search` | The built-in Web Search tool | +| `projects` | Customer-created Projects (OpenAI-compatible) for isolation and cost tracking | +| `workspaces` | Workspaces (Anthropic-compatible) referenced by `anthropic-workspace-id` | +| `mantle_only_model_or_api` | The model, or the API for that model, is listed on `bedrock-mantle` only | +| `in_region_processing` | Strict in-Region processing where `bedrock-runtime` offers only a CRIS profile for that model | + +A `mantle_*` row without a `mantle_selector`, or with a reason outside this table, is a design +defect: rewrite the row as the `bedrock-runtime` path (`runtime_openai_cris` with the source +app's `required_client_api`, or `runtime_messages`) or justify it with one of the values above. +A `runtime_*` or `converse` row never carries `mantle_selector`. + +## Public comparison (read 2026-11-07) + +| Item | `bedrock-runtime` | `bedrock-mantle` | Source page | +| ---------------------------------------------- | --------------------------------------------------------------------------------- | --------------------------------------------------------------- | ----------------------------------------------- | +| Recommended default | Yes — "for most new applications" | Only for capabilities available only there | `endpoints.html`, `models-endpoint-availability.html` | +| InvokeModel / Converse | Supported | Not supported | `endpoints.html` | +| OpenAI Responses API | Supported at `/openai/v1`; always synchronous; default project only | Supported at `/v1` (some models `/openai/v1`); background OK | `inference-responses-api.html` | +| OpenAI Chat Completions API | Supported at `/openai/v1/chat/completions` | Supported at `/v1/chat/completions` (some models `/openai/v1`) | `inference-chat-completions.html` | +| Anthropic Messages API | Supported at `/anthropic` (SDK sends `/anthropic/v1/messages`) | Supported at `/anthropic/v1/messages` | `inference-messages-api.html` | +| Background Responses (`background=true`) | Rejected (400) | Supported | `endpoints.html`, `inference-responses-api.html` | +| Server-side tool use / pre-configured tools | Not available; client-side tools work | Supported | `tool-use-server-side.html`, `endpoints.html` | +| Web Search | Not available | Supported | `web-search.html`, `endpoints.html` | +| Projects / Workspaces | Default project only; use inference profiles for isolation and tagging | Create and manage your own | `projects.html`, `workspaces.html` | +| Cross-Region inference (Geo / Global profiles) | Supported | Not supported — in-Region only | `endpoints.html`, `monitoring-mantle-metrics.html` | +| In-Region processing | Through an on-demand (unprefixed) model ID where the model card lists one; proprietary GPT models list CRIS profiles only | Always in-Region | `endpoints.html`, the per-model cards | +| Guardrails, intelligent prompt routing | Supported | Not supported | `endpoints.html` | +| Application inference profiles / per-request metadata | InvokeModel and Converse only (Responses and Chat Completions reject an AIP target with 400) | Not available — Projects / Workspaces attribute usage | `cost-mgmt-application-inference-profiles.html`, `endpoints.html` | +| Model invocation logging | Supported, including the OpenAI-compatible APIs on this endpoint | Not currently captured | `model-invocation-logging.html` | +| CloudWatch metrics namespace | `AWS/Bedrock` (`Invocations`, `InputTokenCount`, …) | `AWS/BedrockMantle` (`Inferences`, `TotalInputTokens`, …) | `monitoring-mantle-metrics.html` | +| Authentication | SigV4 or Bedrock API key / short-term bearer token; IAM `bedrock:InvokeModel*` | SigV4 or Bedrock API key; IAM `bedrock-mantle:CreateInference` | `endpoints.html` | + +## Lifecycle wording (the only sentence to repeat) + +Existing `bedrock-mantle` applications continue to be fully supported and do not need to change. +Do not add claims about permanence, model-quality equivalence between the endpoints, +launch-day parity for new models, or future plans — the public pages make none, and the +guides in this plugin must not either. Both endpoint surfaces use the same underlying inference +engine (`endpoints.html`). + +## Quotas (why endpoint choice affects capacity planning) + +- **`bedrock-runtime`** — per-model, per-Region token quotas (input and output counted + together, output weighted by a model-specific burndown rate) shared across every inference + API on the endpoint; a requests-per-minute quota exists for some models and not others + (`quotas-runtime.html`, `scaling-throughput-best-practices.html`). +- **`bedrock-mantle`** — no requests-per-minute quota; separate input-tokens-per-minute and + output-tokens-per-minute quotas per model and Region for models with published quotas. A + model without a published quota is not unlimited — internal service capacity and transient + rate limiting still apply (`quotas-mantle.html`, `scaling-throughput-best-practices.html`). +- Traffic on the two endpoints consumes separate quota allocations even for the same model; + plan capacity per endpoint. Batch inference, Provisioned Throughput and custom inference + profiles exist only on `bedrock-runtime`. + +## Sources + +All pages under `https://docs.aws.amazon.com/bedrock/latest/userguide/`, read 2026-11-07: + +- `endpoints.html` — https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html +- `models-endpoint-availability.html` +- `inference-responses-api.html` +- `inference-chat-completions.html` +- `inference-messages-api.html` +- `tool-use-server-side.html` +- `web-search.html` +- `projects.html` +- `workspaces.html` +- `cost-mgmt-application-inference-profiles.html` +- `model-invocation-logging.html` +- `monitoring-mantle-metrics.html` +- `quotas-runtime.html` +- `quotas-mantle.html` +- `scaling-throughput-best-practices.html` diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/anthropic-bedrock.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/anthropic-bedrock.json index eba76d8ce..1f5827190 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/anthropic-bedrock.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/anthropic-bedrock.json @@ -3,7 +3,7 @@ "provider": "anthropic", "verified_at": "2026-09-24", "verified_region": "us-east-1", - "source": "Opus 5.5: AWS Bedrock model card retrieved 2026-09-24, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html. Other entries retain their explicitly dated evidence from the superseded 2026-07-21 catalog (removed; see git history).", + "source": "Opus 5.5: AWS Bedrock model card retrieved 2026-09-24, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html. Other entries retain their explicitly dated evidence from the superseded 2026-07-21 catalog (removed; see git history). The runtime_messages path definition and its rows on claude_opus_5_5, claude_sonnet_5 and claude_haiku_4_5 were added from the three public model cards and https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html on 2026-11-07; earlier rows and their evidence are unchanged.", "disclaimer": "Point-in-time model-card evidence, not an account invocation. Re-probe the chosen model, API and inference profile in the target account and region. Opus 5.5 is Active, uses always-on adaptive thinking, and has no Batch tier. Global profiles are unavailable in GovCloud. Unchanged prior-generation entries retain their earlier evidence dates; Fable and Mythos are excluded from default selection. Opus 5.5 is the only Opus model in the automatic selection pool; older Opus snapshots and pricing records are historical references. The runtime in-region exception in eu-west-2 was verified 2026-10-03; the original catalog and other model evidence dates remain unchanged.", "paths": { "mantle_messages": { @@ -55,6 +55,24 @@ "Structured-output strategy is model-specific: Opus 5.5 rejects forced any/tool choice and disabled thinking; use auto tool choice or prompted JSON with application validation and bounded retries, without a native schema guarantee.", "Availability must be probed in the target account and region" ] + }, + "runtime_messages": { + "sdk": "anthropic", + "client": "Anthropic(base_url=https://bedrock-runtime.{region}.amazonaws.com/anthropic)", + "iam_action": "bedrock:InvokeModel", + "iam_auth": "bedrock_bearer_token_or_sigv4", + "model_id_strategy": "cris_inference_profile", + "supports_cris": true, + "strengths": [ + "Preserves the Anthropic Messages API on the default bedrock-runtime endpoint (bedrock-endpoint-policy.md); the SDK sends /anthropic/v1/messages and adds the anthropic-version header automatically (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html, read 2026-11-07)", + "IAM (bedrock:InvokeModel), model invocation logging and CloudWatch AWS/Bedrock metrics apply, as on every bedrock-runtime path" + ], + "constraints": [ + "Default project only — Workspaces are available only with the Messages API on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/workspaces.html, read 2026-11-07)", + "The model ID is a Global or geography-scoped CRIS inference profile except where the card lists a runtime in-region form (Opus 5.5 in eu-west-2)", + "Structured-output strategy is model-specific: Opus 5.5 rejects forced any/tool choice and disabled thinking; use auto tool choice or prompted JSON with application validation and bounded retries, without a native schema guarantee.", + "Availability must be probed in the target account and region" + ] } }, "models": { @@ -164,6 +182,12 @@ "available": true, "model_id": "anthropic.claude-opus-5-5", "requires_cris": true + }, + "runtime_messages": { + "available": true, + "model_id": "anthropic.claude-opus-5-5", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html (read 2026-11-07): bedrock-runtime Messages supported; Geo ids us./eu./au./jp.anthropic.claude-opus-5-5, Global id global.anthropic.claude-opus-5-5; in-region bare id in eu-west-2 only" } }, "forced_tool_choice_supported": false, @@ -202,6 +226,12 @@ "available": true, "model_id": "anthropic.claude-sonnet-5", "requires_cris": true + }, + "runtime_messages": { + "available": true, + "model_id": "anthropic.claude-sonnet-5", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-5.html (read 2026-11-07): bedrock-runtime Messages supported; Geo ids us./eu./au./in.anthropic.claude-sonnet-5, Global id global.anthropic.claude-sonnet-5; no in-region form" } } }, @@ -231,6 +261,12 @@ "available": true, "model_id": "anthropic.claude-haiku-4-5-20251001-v1:0", "requires_cris": true + }, + "runtime_messages": { + "available": true, + "model_id": "anthropic.claude-haiku-4-5-20251001-v1:0", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-4-5.html (read 2026-11-07): bedrock-runtime Messages supported; Geo ids us./eu./au./jp./in.anthropic.claude-haiku-4-5-20251001-v1:0, Global id global.anthropic.claude-haiku-4-5-20251001-v1:0; no in-region form" } } } diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json index cf642646e..ba77539d3 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json @@ -3,7 +3,7 @@ "provider": "openai", "verified_at": "2026-09-24", "verified_region": "us-east-2", - "source": "Opus 5.5 cross-family target: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html (retrieved 2026-09-24). GPT-5.5 and GPT-5.4 rows re-read from their public model cards on 2026-11-06 (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html): both now list bedrock-runtime (Responses, Chat Completions, Converse via us./global. CRIS ids) and Chat Completions on bedrock-mantle. Other OpenAI entries retain their evidence from the superseded 2026-08-21 catalog (removed; see git history).", + "source": "Opus 5.5 cross-family target: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html (retrieved 2026-09-24). GPT-5.5 and GPT-5.4 rows re-read from their public model cards on 2026-11-06 (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html): both now list bedrock-runtime (Responses, Chat Completions, Converse via us./global. CRIS ids) and Chat Completions on bedrock-mantle. Other OpenAI entries retain their evidence from the superseded 2026-08-21 catalog (removed; see git history). The runtime_openai_responses and runtime_openai_chat path definitions and their per-model rows on every openai_gpt_* entry were added from the nine public model cards and the endpoints / inference-responses-api / inference-chat-completions pages on 2026-11-07; earlier rows and their evidence are unchanged.", "disclaimer": "The Opus cross-family target is updated to Opus 5.5; the GPT-5.5 and GPT-5.4 entries were re-read from their public model cards on 2026-11-06 (runtime CRIS paths, Chat Completions on mantle, 1.05M context, long-context pricing); the other same-model OpenAI paths and their earlier evidence remain unchanged. This is a migration target catalog, not an account-availability claim. Verify the chosen model, API, region and CRIS profile before implementation. Opus 5.5 uses CRIS on runtime outside eu-west-2, has 1M context/128K output, always-on adaptive thinking, and no Batch tier. Fable and Mythos remain excluded from default selection. Opus 5.5 is the only Opus model in the automatic selection pool; older Opus snapshots and pricing records are historical references. The runtime in-region exception in eu-west-2 was verified 2026-10-03; the original catalog and other model evidence dates remain unchanged.", "paths": { "mantle_openai_responses": { @@ -50,7 +50,7 @@ "supports_cris": true, "strengths": [ "Bedrock-native multi-model request surface", - "Supports Bedrock Guardrails (Converse API only), invocation logging, and CloudWatch", + "Supports Bedrock Guardrails (Converse API only) and model invocation logging — both bedrock-runtime-only; CloudWatch metrics exist on both endpoints (AWS/Bedrock on runtime, AWS/BedrockMantle on Mantle; https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-mantle-metrics.html, read 2026-11-07)", "This is a SAME-MODEL governance path via CRIS. GPT-5.6 Terra and Luna publish us./in./global. ids; us./global. only for GPT-5.6 Sol, GPT-5.5 and GPT-5.4 (their model cards list no in. profile; cards read 2026-11-07). The base URL for the OpenAI-compatible APIs is bedrock-runtime.{region}.amazonaws.com/openai/v1. The model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" ], "constraints": [ @@ -60,6 +60,53 @@ "Bedrock-native models generally require a Global or geography-scoped CRIS inference profile", "Availability must be probed in the target account and region" ] + }, + "runtime_openai_responses": { + "sdk": "openai", + "client": "OpenAI(base_url=bedrock-runtime)", + "endpoint": "https://bedrock-runtime.{region}.amazonaws.com/openai/v1", + "api": "responses", + "iam_action": "bedrock:InvokeModel", + "iam_auth": "bedrock_bearer_token_or_sigv4", + "model_id_strategy": "cris_inference_profile", + "supports_cris": true, + "strengths": [ + "Default endpoint per bedrock-endpoint-policy.md: AWS recommends bedrock-runtime for most new applications (https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html, read 2026-11-07)", + "Keeps the OpenAI SDK and the source Responses API — only base_url and api_key change (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, read 2026-11-07)", + "Model invocation logging (including the OpenAI-compatible APIs) and CloudWatch AWS/Bedrock metrics apply (https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html, read 2026-11-07)", + "SAME-MODEL path whose model ID is the CRIS inference profile: us. or global. ids; in. additionally for GPT-5.6 Terra and Luna only (model cards read 2026-11-07)" + ], + "constraints": [ + "Always synchronous: background=true is rejected with a 400 error; store keeps its default (https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html, read 2026-11-07)", + "Default project only — additional Projects are bedrock-mantle only (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, read 2026-11-07)", + "No Web Search or other server-side/pre-configured tools on bedrock-runtime; those are bedrock-mantle selectors per bedrock-endpoint-policy.md", + "Guardrails and application inference profiles are Converse-only on the GPT-5.6 Sol/Terra/Luna, GPT-5.5, GPT-5.4 and GPT-6 Astra cards; the GPT-6 Sol, GPT-6 Luna and GPT-6.1 Sol cards list wider Guardrails API scopes while application inference profiles stay InvokeModel/Converse-only — read the selected card (cards read 2026-11-07)", + "IAM needs bedrock:InvokeModel on the inference profile and on the account's default project (arn:aws:bedrock:{region}:{account-id}:project/default)", + "Availability must be probed in the target account and region" + ] + }, + "runtime_openai_chat": { + "sdk": "openai", + "client": "OpenAI(base_url=bedrock-runtime)", + "endpoint": "https://bedrock-runtime.{region}.amazonaws.com/openai/v1", + "api": "chat_completions", + "iam_action": "bedrock:InvokeModel", + "iam_auth": "bedrock_bearer_token_or_sigv4", + "model_id_strategy": "cris_inference_profile", + "supports_cris": true, + "strengths": [ + "Default endpoint per bedrock-endpoint-policy.md: AWS recommends bedrock-runtime for most new applications (https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html, read 2026-11-07)", + "Keeps the OpenAI SDK and the source Chat Completions API at /openai/v1/chat/completions — only base_url and api_key change (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions.html, read 2026-11-07)", + "Model invocation logging (including the OpenAI-compatible APIs) and CloudWatch AWS/Bedrock metrics apply (https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html, read 2026-11-07)", + "SAME-MODEL path whose model ID is the CRIS inference profile: us. or global. ids; in. additionally for GPT-5.6 Terra and Luna only (model cards read 2026-11-07)" + ], + "constraints": [ + "Always synchronous and default project only — additional Projects, background inference and server-side/pre-configured tools (including Web Search) are bedrock-mantle only (https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html, read 2026-11-07)", + "Guardrails and application inference profiles are Converse-only on the GPT-5.6 Sol/Terra/Luna, GPT-5.5, GPT-5.4 and GPT-6 Astra cards; the GPT-6 Sol, GPT-6 Luna and GPT-6.1 Sol cards list wider Guardrails API scopes while application inference profiles stay InvokeModel/Converse-only — read the selected card (cards read 2026-11-07)", + "Structured outputs and prompt caching on Chat Completions are model-specific: the GPT-5.5 and GPT-5.4 cards list both as Responses-only on bedrock-runtime; the GPT-5.6 and GPT-6 cards list Chat Completions for structured outputs and implicit caching (cards read 2026-11-07)", + "IAM needs bedrock:InvokeModel on the inference profile and on the account's default project (arn:aws:bedrock:{region}:{account-id}:project/default)", + "Availability must be probed in the target account and region" + ] } }, "models": { @@ -103,6 +150,18 @@ "model_id": "openai.gpt-5.6-sol", "requires_cris": true, "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x" + }, + "runtime_openai_responses": { + "available": true, + "model_id": "openai.gpt-5.6-sol", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-07): bedrock-runtime lists Responses supported (base URL bedrock-runtime.{region}.amazonaws.com/openai/v1); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol" + }, + "runtime_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.6-sol", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported (/openai/v1/chat/completions); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol" } } }, @@ -153,6 +212,18 @@ "model_id": "openai.gpt-5.5", "requires_cris": true, "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.5 (Geo) or global.openai.gpt-5.5 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs" + }, + "runtime_openai_responses": { + "available": true, + "model_id": "openai.gpt-5.5", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-07): bedrock-runtime lists Responses supported; structured outputs and implicit prompt caching are Responses-only on runtime; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" + }, + "runtime_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.5", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; the card scopes structured outputs and implicit prompt caching to Responses; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" } } }, @@ -193,6 +264,18 @@ "model_id": "openai.gpt-5.6-terra", "requires_cris": true, "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x" + }, + "runtime_openai_responses": { + "available": true, + "model_id": "openai.gpt-5.6-terra", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-07): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra" + }, + "runtime_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.6-terra", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra" } } }, @@ -233,6 +316,18 @@ "model_id": "openai.gpt-5.6-luna", "requires_cris": true, "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x" + }, + "runtime_openai_responses": { + "available": true, + "model_id": "openai.gpt-5.6-luna", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-07): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna" + }, + "runtime_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.6-luna", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna" } } }, @@ -286,6 +381,18 @@ "model_id": "openai.gpt-5.4", "requires_cris": true, "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.4 (Geo) or global.openai.gpt-5.4 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs" + }, + "runtime_openai_responses": { + "available": true, + "model_id": "openai.gpt-5.4", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-07): bedrock-runtime lists Responses supported; structured outputs and implicit prompt caching are Responses-only on runtime; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" + }, + "runtime_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.4", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; the card scopes structured outputs and implicit prompt caching to Responses; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" } } }, @@ -427,6 +534,126 @@ "structured_output_json" ], "evidence": "AWS model card, runtime JSON Schema reverified 2026-10-03: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html. Runtime APIs: Responses, Chat Completions, Converse; Invoke and Messages unsupported. US Geo/Global CRIS only, no in-region or EU/India Geo form. Guardrails and application inference profiles are Converse-only. Non-streaming JSON Schema outputs supported: Converse requires outputConfig.textFormat.structure.jsonSchema and additionalModelRequestFields.text.format.strict=true. Streaming structured output and server-side tool use are not evidenced. Runtime TPM accounting uses 10x output-token burndown." + }, + "runtime_openai_responses": { + "available": true, + "model_id": "openai.gpt-6-astra", + "requires_cris": true, + "supported_regions": [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "ca-central-1", + "eu-central-1", + "eu-north-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "sa-east-1" + ], + "inference_profile_regions": { + "us.openai.gpt-6-astra": [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "ca-central-1" + ], + "global.openai.gpt-6-astra": [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "ca-central-1", + "eu-central-1", + "eu-north-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "sa-east-1" + ] + }, + "capabilities": [ + "reasoning", + "image_input_vision", + "streaming", + "structured_output_json", + "prompt_caching" + ], + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html (read 2026-11-07): bedrock-runtime lists Responses supported; runtime features: response streaming, structured outputs (JSON Schema), implicit and explicit prompt caching on Responses, reasoning summaries; application inference profiles are InvokeModel/Converse-only and server-side tool use is unsupported; no in-region form; Geo id us.openai.gpt-6-astra, Global id global.openai.gpt-6-astra" + }, + "runtime_openai_chat": { + "available": true, + "model_id": "openai.gpt-6-astra", + "requires_cris": true, + "supported_regions": [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "ca-central-1", + "eu-central-1", + "eu-north-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "sa-east-1" + ], + "inference_profile_regions": { + "us.openai.gpt-6-astra": [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "ca-central-1" + ], + "global.openai.gpt-6-astra": [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "ca-central-1", + "eu-central-1", + "eu-north-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "sa-east-1" + ] + }, + "capabilities": [ + "reasoning", + "image_input_vision", + "streaming", + "structured_output_json", + "prompt_caching" + ], + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; runtime features: response streaming, structured outputs (JSON Schema), implicit and explicit prompt caching on Chat Completions; application inference profiles are InvokeModel/Converse-only and server-side tool use is unsupported; no in-region form; Geo id us.openai.gpt-6-astra, Global id global.openai.gpt-6-astra" } } }, @@ -452,7 +679,9 @@ "paths": { "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported."} + "runtime_converse": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported."}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-07): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"} } }, "openai_gpt_6_luna": { @@ -477,7 +706,9 @@ "paths": { "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke."} + "runtime_converse": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke."}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-07): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"} } }, "openai_gpt_6_1_sol": { @@ -502,7 +733,9 @@ "paths": { "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported."} + "runtime_converse": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported."}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-07): bedrock-runtime lists Responses supported; Guardrails are listed for Chat Completions/InvokeModel/Converse but not Responses, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; Guardrails listed on Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"} } }, "anthropic_claude_sonnet_5": { diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-recommendation-input.json b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-recommendation-input.json index 614779010..b87015444 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-recommendation-input.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-recommendation-input.json @@ -155,7 +155,21 @@ "mantle_openai_chat", "mantle_openai_responses", "runtime_converse", - "runtime_invoke" + "runtime_invoke", + "runtime_messages", + "runtime_openai_chat", + "runtime_openai_responses" + ] + }, + "mantle_selector": { + "enum": [ + "background_inference", + "server_side_tools", + "web_search", + "projects", + "workspaces", + "mantle_only_model_or_api", + "in_region_processing" ] }, "min_context_tokens": { @@ -170,7 +184,7 @@ "type": "boolean" }, "data_residency": { - "enum": ["global_allowed", "geo_required", "unknown"] + "enum": ["global_allowed", "geo_required", "in_region_required", "unknown"] }, "cris_geography": { "enum": ["us", "eu", "jp", "au"] diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-recommendation.json b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-recommendation.json index 5976cb41c..56f24e7a1 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-recommendation.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-recommendation.json @@ -26,7 +26,22 @@ "mantle_openai_chat", "mantle_openai_responses", "runtime_converse", - "runtime_invoke" + "runtime_invoke", + "runtime_messages", + "runtime_openai_chat", + "runtime_openai_responses" + ] + }, + "mantleSelector": { + "enum": [ + "background_inference", + "server_side_tools", + "web_search", + "projects", + "workspaces", + "mantle_only_model_or_api", + "in_region_processing", + null ] }, "source": { @@ -128,6 +143,9 @@ }, "reason": { "type": "string" + }, + "mantle_selector": { + "$ref": "#/$defs/mantleSelector" } }, "additionalProperties": false @@ -369,6 +387,9 @@ "invocation_model_id": { "type": ["string", "null"] }, + "mantle_selector": { + "$ref": "#/$defs/mantleSelector" + }, "decision_options": { "type": "array", "items": { diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-verification.json b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-verification.json index 1afd3cbb1..4d5f19b49 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-verification.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-verification.json @@ -7,7 +7,10 @@ "mantle_openai_chat", "mantle_openai_responses", "runtime_converse", - "runtime_invoke" + "runtime_invoke", + "runtime_messages", + "runtime_openai_chat", + "runtime_openai_responses" ] }, "verification": { diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_catalog_paths.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_catalog_paths.py new file mode 100644 index 000000000..d5855deb0 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_catalog_paths.py @@ -0,0 +1,128 @@ +"""Catalog-shape guards for the Bedrock path vocabulary. + +The two vendored model catalogs, the recommendation input/output schemas and +the verification schema must agree on one set of API path names. The +bedrock-runtime paths (runtime_openai_responses, runtime_openai_chat, +runtime_messages) were added from the public model cards on 2026-11-07; every +such row must cite its docs.aws.amazon.com source and a read date. +""" +import json +import pathlib +import re + +import pytest + +import model_recommendation + + +SCHEMAS = pathlib.Path(model_recommendation.__file__).parent / "schemas" +RUNTIME_SDK_PATHS = ("runtime_openai_responses", "runtime_openai_chat", "runtime_messages") + + +def _schema(name): + return json.loads((SCHEMAS / name).read_text()) + + +@pytest.fixture(scope="module") +def catalogs(): + return { + "anthropic": model_recommendation.load_catalog(), + "openai": model_recommendation.load_openai_catalog(), + } + + +def test_model_path_keys_are_declared_at_the_top_level(catalogs): + for name, catalog in catalogs.items(): + declared = set(catalog["paths"]) + for model_key, model in catalog["models"].items(): + undeclared = set(model["paths"]) - declared + assert not undeclared, f"{name}: {model_key} uses undeclared paths {sorted(undeclared)}" + + +def test_catalog_paths_match_every_schema_enum(catalogs): + catalog_paths = set() + for catalog in catalogs.values(): + catalog_paths |= set(catalog["paths"]) + output_enum = set(_schema("model-recommendation.json")["$defs"]["apiPath"]["enum"]) + verification_enum = set(_schema("model-verification.json")["$defs"]["apiPath"]["enum"]) + input_schema = _schema("model-recommendation-input.json") + requirements = input_schema["properties"]["workloads"]["items"]["properties"]["requirements"] + input_enum = set(requirements["properties"]["preferred_api_path"]["enum"]) + + assert catalog_paths == output_enum + assert output_enum == verification_enum + assert output_enum == input_enum + + +def test_every_path_row_has_the_required_keys(catalogs): + for name, catalog in catalogs.items(): + for model_key, model in catalog["models"].items(): + for path, row in model["paths"].items(): + for key in ("available", "model_id", "requires_cris"): + assert key in row, f"{name}: {model_key}.{path} lacks {key}" + assert isinstance(row["available"], bool) + assert isinstance(row["requires_cris"], bool) + assert row["model_id"] + + +def test_openai_runtime_rows_are_cris_profiles_on_openai_ids(catalogs): + for model_key, model in catalogs["openai"]["models"].items(): + if not model_key.startswith("openai_gpt_"): + continue + for path in ("runtime_openai_responses", "runtime_openai_chat"): + assert path in model["paths"], f"{model_key} lacks {path}" + for path, row in model["paths"].items(): + if path.startswith("runtime_"): + assert row["requires_cris"] is True, f"{model_key}.{path}" + assert row["model_id"].startswith("openai."), f"{model_key}.{path}" + + +def test_openai_runtime_sdk_rows_do_not_appear_on_claude_entries(catalogs): + for model_key, model in catalogs["openai"]["models"].items(): + if model_key.startswith("anthropic_claude_"): + for path in ("runtime_openai_responses", "runtime_openai_chat"): + assert path not in model["paths"], f"{model_key} must not carry {path}" + + +def test_anthropic_runtime_messages_uses_the_runtime_converse_model_id(catalogs): + for model_key, model in catalogs["anthropic"]["models"].items(): + paths = model["paths"] + assert "runtime_messages" in paths, f"{model_key} lacks runtime_messages" + assert paths["runtime_messages"]["model_id"] == paths["runtime_converse"]["model_id"], model_key + assert paths["runtime_messages"]["requires_cris"] is True, model_key + + +def test_runtime_sdk_path_definitions_target_bedrock_runtime(catalogs): + openai_paths = catalogs["openai"]["paths"] + for path in ("runtime_openai_responses", "runtime_openai_chat"): + definition = openai_paths[path] + assert definition["endpoint"] == "https://bedrock-runtime.{region}.amazonaws.com/openai/v1" + assert definition["iam_action"] == "bedrock:InvokeModel" + assert definition["model_id_strategy"] == "cris_inference_profile" + assert definition["supports_cris"] is True + assert openai_paths["runtime_openai_responses"]["api"] == "responses" + assert openai_paths["runtime_openai_chat"]["api"] == "chat_completions" + messages = catalogs["anthropic"]["paths"]["runtime_messages"] + assert "bedrock-runtime.{region}.amazonaws.com/anthropic" in messages["client"] + assert messages["iam_action"] == "bedrock:InvokeModel" + assert messages["model_id_strategy"] == "cris_inference_profile" + assert messages["supports_cris"] is True + + +def test_openai_runtime_converse_strengths_name_both_cloudwatch_namespaces(catalogs): + text = "\n".join(catalogs["openai"]["paths"]["runtime_converse"]["strengths"]) + assert "AWS/BedrockMantle" in text + assert "monitoring-mantle-metrics" in text + assert "invocation logging, and CloudWatch" not in text + + +def test_new_runtime_rows_cite_public_docs_with_a_read_date(catalogs): + read_date = re.compile(r"read 20\d\d-\d\d-\d\d") + for name, catalog in catalogs.items(): + for model_key, model in catalog["models"].items(): + for path, row in model["paths"].items(): + if path not in RUNTIME_SDK_PATHS: + continue + evidence = row.get("evidence", "") + assert "docs.aws.amazon.com" in evidence, f"{name}: {model_key}.{path}" + assert read_date.search(evidence), f"{name}: {model_key}.{path} lacks a read date" diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py index 96f53ce3b..4d909e28b 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py @@ -1,5 +1,7 @@ import json import pathlib +import sys +import types import jsonschema import pytest @@ -679,3 +681,249 @@ def factory(region): assert verification["status"] == "not_applicable" assert called["n"] == 0 # no client constructed for a non-selected recommendation + + +# --- bedrock-runtime Responses / Chat Completions / Messages probes ----------- +# +# The engines do not yet emit these paths, so the recommendations are built by +# hand from a recommended result and re-pointed at the Runtime path. + +RECOMMENDATION_SCHEMA = json.loads( + (pathlib.Path(verify_model_path.__file__).parent / "schemas" / "model-recommendation.json").read_text() +) +VERIFICATION_SCHEMA = json.loads( + (pathlib.Path(verify_model_path.__file__).parent / "schemas" / "model-verification.json").read_text() +) +API_PATHS = RECOMMENDATION_SCHEMA["$defs"]["apiPath"]["enum"] + + +def _runtime_recommendation(provider, path, path_model_id, invocation_model_id): + if provider == "openai": + data = _openai_input({"api_continuity": "required"}) + else: + data = _input() + recommendation = model_recommendation.recommend(data) + workload = next(iter(recommendation["workloads"].values())) + assert workload["decision_status"] == "recommended" + workload["api_path"] = path + workload["invocation_model_id"] = invocation_model_id + workload["primary_model"] = path_model_id + workload["model_identity"]["path_model_id"] = path_model_id + workload["model_identity"]["requires_cris"] = True + workload["verification"]["invocation_model_id"] = invocation_model_id + jsonschema.validate(recommendation, RECOMMENDATION_SCHEMA) + return recommendation + + +def test_runtime_responses_probe_calls_responses_create_with_exact_cris_id(): + client = FakeOpenAIClient(FakeResponses(response={"model": "global.openai.gpt-5.4"})) + recommendation = _runtime_recommendation( + "openai", "runtime_openai_responses", "openai.gpt-5.4", "global.openai.gpt-5.4" + ) + result = verify_model_path.verify_recommendation( + recommendation, now=NOW, + runtime_openai_responses_client_factory=lambda region: client, + ) + jsonschema.validate(result, VERIFICATION_SCHEMA, format_checker=jsonschema.FormatChecker()) + verification = result["workloads"]["openai-svc"] + + assert verification["status"] == "passed" + assert verification["api_path"] == "runtime_openai_responses" + assert verification["response_model_id"] == "global.openai.gpt-5.4" + assert client.responses.calls == [{"model": "global.openai.gpt-5.4", "input": verify_model_path.PROMPT}] + assert client.chat.completions.calls == [] + + +def test_runtime_responses_failure_does_not_substitute_model(): + client = FakeOpenAIClient(FakeResponses(error=PermissionError("AccessDeniedException"))) + recommendation = _runtime_recommendation( + "openai", "runtime_openai_responses", "openai.gpt-5.4", "global.openai.gpt-5.4" + ) + result = verify_model_path.verify_recommendation( + recommendation, now=NOW, + runtime_openai_responses_client_factory=lambda region: client, + ) + jsonschema.validate(result, VERIFICATION_SCHEMA, format_checker=jsonschema.FormatChecker()) + verification = result["workloads"]["openai-svc"] + + assert verification["status"] == "failed" + assert verification["error"] == {"type": "PermissionError", "message": "AccessDeniedException"} + assert verification["invocation_model_id"] == "global.openai.gpt-5.4" + assert [call["model"] for call in client.responses.calls] == ["global.openai.gpt-5.4"] + + +def test_runtime_chat_probe_calls_chat_completions_with_exact_cris_id(): + completions = FakeCompletions(response={"model": "us.openai.gpt-6-sol"}) + client = FakeOpenAIClient(chat=FakeChat(completions)) + recommendation = _runtime_recommendation( + "openai", "runtime_openai_chat", "openai.gpt-6-sol", "us.openai.gpt-6-sol" + ) + result = verify_model_path.verify_recommendation( + recommendation, now=NOW, + runtime_openai_chat_client_factory=lambda region: client, + ) + jsonschema.validate(result, VERIFICATION_SCHEMA, format_checker=jsonschema.FormatChecker()) + verification = result["workloads"]["openai-svc"] + + assert verification["status"] == "passed" + assert verification["api_path"] == "runtime_openai_chat" + assert verification["response_model_id"] == "us.openai.gpt-6-sol" + assert completions.calls == [ + { + "model": "us.openai.gpt-6-sol", + "max_completion_tokens": 8, + "messages": [{"role": "user", "content": verify_model_path.PROMPT}], + } + ] + assert client.responses.calls == [] + + +def test_runtime_chat_failure_does_not_substitute_model(): + completions = FakeCompletions(error=RuntimeError("access denied")) + client = FakeOpenAIClient(chat=FakeChat(completions)) + recommendation = _runtime_recommendation( + "openai", "runtime_openai_chat", "openai.gpt-6-sol", "us.openai.gpt-6-sol" + ) + result = verify_model_path.verify_recommendation( + recommendation, now=NOW, + runtime_openai_chat_client_factory=lambda region: client, + ) + verification = result["workloads"]["openai-svc"] + + assert verification["status"] == "failed" + assert verification["error"] == {"type": "RuntimeError", "message": "access denied"} + assert verification["invocation_model_id"] == "us.openai.gpt-6-sol" + assert [call["model"] for call in completions.calls] == ["us.openai.gpt-6-sol"] + + +def test_runtime_messages_probe_calls_messages_create_with_exact_cris_id(): + messages = FakeMessages(response={"model": "global.anthropic.claude-sonnet-5"}) + client = FakeMantleClient(messages) + recommendation = _runtime_recommendation( + "anthropic", "runtime_messages", "anthropic.claude-sonnet-5", "global.anthropic.claude-sonnet-5" + ) + result = verify_model_path.verify_recommendation( + recommendation, now=NOW, + runtime_messages_client_factory=lambda region: client, + ) + jsonschema.validate(result, VERIFICATION_SCHEMA, format_checker=jsonschema.FormatChecker()) + verification = result["workloads"]["support-agent"] + + assert verification["status"] == "passed" + assert verification["api_path"] == "runtime_messages" + assert verification["response_model_id"] == "global.anthropic.claude-sonnet-5" + assert messages.calls == [ + { + "model": "global.anthropic.claude-sonnet-5", + "max_tokens": 8, + "messages": [{"role": "user", "content": verify_model_path.PROMPT}], + } + ] + + +def test_runtime_messages_failure_does_not_substitute_model(): + messages = FakeMessages(error=PermissionError("denied")) + recommendation = _runtime_recommendation( + "anthropic", "runtime_messages", "anthropic.claude-sonnet-5", "global.anthropic.claude-sonnet-5" + ) + result = verify_model_path.verify_recommendation( + recommendation, now=NOW, + runtime_messages_client_factory=lambda region: FakeMantleClient(messages), + ) + verification = result["workloads"]["support-agent"] + + assert verification["status"] == "failed" + assert verification["error"] == {"type": "PermissionError", "message": "denied"} + assert verification["invocation_model_id"] == "global.anthropic.claude-sonnet-5" + assert [call["model"] for call in messages.calls] == ["global.anthropic.claude-sonnet-5"] + + +def test_runtime_factories_do_not_fall_back_to_mantle_factories(): + mantle_calls = [] + recommendation = _runtime_recommendation( + "openai", "runtime_openai_responses", "openai.gpt-5.4", "global.openai.gpt-5.4" + ) + runtime_client = FakeOpenAIClient() + verify_model_path.verify_recommendation( + recommendation, now=NOW, + openai_responses_client_factory=lambda region: mantle_calls.append(region), + openai_chat_client_factory=lambda region: mantle_calls.append(region), + runtime_openai_responses_client_factory=lambda region: runtime_client, + ) + assert mantle_calls == [] + assert [call["model"] for call in runtime_client.responses.calls] == ["global.openai.gpt-5.4"] + + +@pytest.mark.parametrize("path", API_PATHS) +def test_every_api_path_enum_value_is_probed(path): + provider = "openai" if "openai" in path else "anthropic" + path_model_id = "openai.gpt-5.4" if provider == "openai" else "anthropic.claude-sonnet-5" + invocation_model_id = f"global.{path_model_id}" + recommendation = _runtime_recommendation(provider, path, path_model_id, invocation_model_id) + result = verify_model_path.verify_recommendation( + recommendation, now=NOW, + mantle_client_factory=lambda region: FakeMantleClient(), + runtime_client_factory=lambda region: FakeRuntimeClient(), + openai_responses_client_factory=lambda region: FakeOpenAIClient(), + openai_chat_client_factory=lambda region: FakeOpenAIClient(), + runtime_openai_responses_client_factory=lambda region: FakeOpenAIClient(), + runtime_openai_chat_client_factory=lambda region: FakeOpenAIClient(), + runtime_messages_client_factory=lambda region: FakeMantleClient(), + ) + jsonschema.validate(result, VERIFICATION_SCHEMA, format_checker=jsonschema.FormatChecker()) + verification = next(iter(result["workloads"].values())) + + assert verification["status"] == "passed", verification["error"] + assert verification["error"] is None + assert verification["api_path"] == path + + +@pytest.mark.parametrize("builder,module,package", [ + (verify_model_path._default_runtime_openai_responses_client, "openai", "openai"), + (verify_model_path._default_runtime_openai_chat_client, "openai", "openai"), + (verify_model_path._default_runtime_messages_client, "anthropic", "anthropic"), +]) +def test_runtime_default_clients_name_the_missing_package(monkeypatch, builder, module, package): + monkeypatch.setitem(sys.modules, module, None) + with pytest.raises(RuntimeError, match=f"requires the {package} package"): + builder("us-east-1") + + +def _install_fake_sdk(monkeypatch, module, class_name, captured): + def constructor(**kwargs): + captured.append(kwargs) + return object() + + fake = types.ModuleType(module) + setattr(fake, class_name, constructor) + monkeypatch.setitem(sys.modules, module, fake) + token_module = types.ModuleType("aws_bedrock_token_generator") + token_module.provide_token = lambda region: f"token-for-{region}" + monkeypatch.setitem(sys.modules, "aws_bedrock_token_generator", token_module) + + +@pytest.mark.parametrize("builder,base_url", [ + (verify_model_path._default_runtime_openai_responses_client, + "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1"), + (verify_model_path._default_runtime_openai_chat_client, + "https://bedrock-runtime.us-west-2.amazonaws.com/openai/v1"), + (verify_model_path._default_openai_responses_client, + "https://bedrock-mantle.us-west-2.api.aws/openai/v1"), + (verify_model_path._default_openai_chat_client, + "https://bedrock-mantle.us-west-2.api.aws/openai/v1"), +]) +def test_openai_default_clients_point_at_their_endpoint(monkeypatch, builder, base_url): + captured = [] + _install_fake_sdk(monkeypatch, "openai", "OpenAI", captured) + builder("us-west-2") + assert captured == [{"base_url": base_url, "api_key": "token-for-us-west-2"}] + + +def test_runtime_messages_default_client_points_at_bedrock_runtime_anthropic(monkeypatch): + captured = [] + _install_fake_sdk(monkeypatch, "anthropic", "Anthropic", captured) + verify_model_path._default_runtime_messages_client("eu-west-2") + assert captured == [{ + "base_url": "https://bedrock-runtime.eu-west-2.amazonaws.com/anthropic", + "api_key": "token-for-eu-west-2", + }] diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/verify_model_path.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/verify_model_path.py index bee09fa7c..eada4fd1c 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/verify_model_path.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/verify_model_path.py @@ -67,8 +67,8 @@ def destinations(profile): raise error -def _default_openai_client(region, path_name): - """Lazily build an OpenAI SDK client pointed at the Bedrock Mantle endpoint. +def _default_openai_client(region, path_name, base_url): + """Lazily build an OpenAI SDK client pointed at a Bedrock OpenAI-compatible base URL. Imports are deferred so the offline recommendation/verify code path never requires the openai or token-generator packages. @@ -77,24 +77,68 @@ def _default_openai_client(region, path_name): from openai import OpenAI except ImportError as exc: raise RuntimeError( - f"Mantle {path_name} verification requires the openai package" + f"{path_name} verification requires the openai package" ) from exc try: from aws_bedrock_token_generator import provide_token except ImportError as exc: raise RuntimeError( - f"Mantle {path_name} verification requires aws-bedrock-token-generator" + f"{path_name} verification requires aws-bedrock-token-generator" ) from exc - base_url = f"https://bedrock-mantle.{region}.api.aws/openai/v1" return OpenAI(base_url=base_url, api_key=provide_token(region=region)) +def _mantle_openai_base_url(region): + return f"https://bedrock-mantle.{region}.api.aws/openai/v1" + + +def _runtime_openai_base_url(region): + return f"https://bedrock-runtime.{region}.amazonaws.com/openai/v1" + + def _default_openai_responses_client(region): - return _default_openai_client(region, "Responses") + return _default_openai_client(region, "Mantle Responses", _mantle_openai_base_url(region)) def _default_openai_chat_client(region): - return _default_openai_client(region, "Chat Completions") + return _default_openai_client( + region, "Mantle Chat Completions", _mantle_openai_base_url(region) + ) + + +def _default_runtime_openai_responses_client(region): + return _default_openai_client( + region, "Runtime Responses", _runtime_openai_base_url(region) + ) + + +def _default_runtime_openai_chat_client(region): + return _default_openai_client( + region, "Runtime Chat Completions", _runtime_openai_base_url(region) + ) + + +def _default_runtime_messages_client(region): + """Lazily build an Anthropic SDK client pointed at bedrock-runtime /anthropic. + + https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html + (read 2026-11-07): the SDK sends /anthropic/v1/messages and adds the + anthropic-version header; the Bedrock bearer token is passed as api_key. + """ + try: + from anthropic import Anthropic + except ImportError as exc: + raise RuntimeError( + "Runtime Messages verification requires the anthropic package" + ) from exc + try: + from aws_bedrock_token_generator import provide_token + except ImportError as exc: + raise RuntimeError( + "Runtime Messages verification requires aws-bedrock-token-generator" + ) from exc + base_url = f"https://bedrock-runtime.{region}.amazonaws.com/anthropic" + return Anthropic(base_url=base_url, api_key=provide_token(region=region)) def _probe_openai_responses(client, model_id): @@ -171,6 +215,9 @@ def verify_workload( openai_chat_client_factory=None, now=None, control_client_factory=None, + runtime_openai_responses_client_factory=None, + runtime_openai_chat_client_factory=None, + runtime_messages_client_factory=None, ): result = _base_result(workload_id, recommendation) if recommendation["decision_status"] != "recommended": @@ -218,6 +265,16 @@ def verify_workload( elif path == "runtime_invoke": factory = runtime_client_factory or _default_runtime_client response = _probe_invoke(factory(result["region"]), model_id) + elif path == "runtime_openai_responses": + factory = (runtime_openai_responses_client_factory + or _default_runtime_openai_responses_client) + response = _probe_openai_responses(factory(result["region"]), model_id) + elif path == "runtime_openai_chat": + factory = runtime_openai_chat_client_factory or _default_runtime_openai_chat_client + response = _probe_openai_chat(factory(result["region"]), model_id) + elif path == "runtime_messages": + factory = runtime_messages_client_factory or _default_runtime_messages_client + response = _probe_mantle(factory(result["region"]), model_id) else: raise ValueError(f"verification is not implemented for API path: {path}") except Exception as exc: @@ -242,6 +299,9 @@ def verify_recommendation( openai_chat_client_factory=None, now=None, control_client_factory=None, + runtime_openai_responses_client_factory=None, + runtime_openai_chat_client_factory=None, + runtime_messages_client_factory=None, ): selected = set(workload_ids or recommendation["workloads"]) unknown = sorted(selected - set(recommendation["workloads"])) @@ -262,6 +322,9 @@ def verify_recommendation( openai_chat_client_factory=openai_chat_client_factory, now=generated_at, control_client_factory=control_client_factory, + runtime_openai_responses_client_factory=runtime_openai_responses_client_factory, + runtime_openai_chat_client_factory=runtime_openai_chat_client_factory, + runtime_messages_client_factory=runtime_messages_client_factory, ) return { "schema_version": 1, diff --git a/plugins/aws-startup-advisor/skills/shared/ai/models/anthropic-bedrock.json b/plugins/aws-startup-advisor/skills/shared/ai/models/anthropic-bedrock.json index eba76d8ce..1f5827190 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/models/anthropic-bedrock.json +++ b/plugins/aws-startup-advisor/skills/shared/ai/models/anthropic-bedrock.json @@ -3,7 +3,7 @@ "provider": "anthropic", "verified_at": "2026-09-24", "verified_region": "us-east-1", - "source": "Opus 5.5: AWS Bedrock model card retrieved 2026-09-24, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html. Other entries retain their explicitly dated evidence from the superseded 2026-07-21 catalog (removed; see git history).", + "source": "Opus 5.5: AWS Bedrock model card retrieved 2026-09-24, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html. Other entries retain their explicitly dated evidence from the superseded 2026-07-21 catalog (removed; see git history). The runtime_messages path definition and its rows on claude_opus_5_5, claude_sonnet_5 and claude_haiku_4_5 were added from the three public model cards and https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html on 2026-11-07; earlier rows and their evidence are unchanged.", "disclaimer": "Point-in-time model-card evidence, not an account invocation. Re-probe the chosen model, API and inference profile in the target account and region. Opus 5.5 is Active, uses always-on adaptive thinking, and has no Batch tier. Global profiles are unavailable in GovCloud. Unchanged prior-generation entries retain their earlier evidence dates; Fable and Mythos are excluded from default selection. Opus 5.5 is the only Opus model in the automatic selection pool; older Opus snapshots and pricing records are historical references. The runtime in-region exception in eu-west-2 was verified 2026-10-03; the original catalog and other model evidence dates remain unchanged.", "paths": { "mantle_messages": { @@ -55,6 +55,24 @@ "Structured-output strategy is model-specific: Opus 5.5 rejects forced any/tool choice and disabled thinking; use auto tool choice or prompted JSON with application validation and bounded retries, without a native schema guarantee.", "Availability must be probed in the target account and region" ] + }, + "runtime_messages": { + "sdk": "anthropic", + "client": "Anthropic(base_url=https://bedrock-runtime.{region}.amazonaws.com/anthropic)", + "iam_action": "bedrock:InvokeModel", + "iam_auth": "bedrock_bearer_token_or_sigv4", + "model_id_strategy": "cris_inference_profile", + "supports_cris": true, + "strengths": [ + "Preserves the Anthropic Messages API on the default bedrock-runtime endpoint (bedrock-endpoint-policy.md); the SDK sends /anthropic/v1/messages and adds the anthropic-version header automatically (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-messages-api.html, read 2026-11-07)", + "IAM (bedrock:InvokeModel), model invocation logging and CloudWatch AWS/Bedrock metrics apply, as on every bedrock-runtime path" + ], + "constraints": [ + "Default project only — Workspaces are available only with the Messages API on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/workspaces.html, read 2026-11-07)", + "The model ID is a Global or geography-scoped CRIS inference profile except where the card lists a runtime in-region form (Opus 5.5 in eu-west-2)", + "Structured-output strategy is model-specific: Opus 5.5 rejects forced any/tool choice and disabled thinking; use auto tool choice or prompted JSON with application validation and bounded retries, without a native schema guarantee.", + "Availability must be probed in the target account and region" + ] } }, "models": { @@ -164,6 +182,12 @@ "available": true, "model_id": "anthropic.claude-opus-5-5", "requires_cris": true + }, + "runtime_messages": { + "available": true, + "model_id": "anthropic.claude-opus-5-5", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html (read 2026-11-07): bedrock-runtime Messages supported; Geo ids us./eu./au./jp.anthropic.claude-opus-5-5, Global id global.anthropic.claude-opus-5-5; in-region bare id in eu-west-2 only" } }, "forced_tool_choice_supported": false, @@ -202,6 +226,12 @@ "available": true, "model_id": "anthropic.claude-sonnet-5", "requires_cris": true + }, + "runtime_messages": { + "available": true, + "model_id": "anthropic.claude-sonnet-5", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-sonnet-5.html (read 2026-11-07): bedrock-runtime Messages supported; Geo ids us./eu./au./in.anthropic.claude-sonnet-5, Global id global.anthropic.claude-sonnet-5; no in-region form" } } }, @@ -231,6 +261,12 @@ "available": true, "model_id": "anthropic.claude-haiku-4-5-20251001-v1:0", "requires_cris": true + }, + "runtime_messages": { + "available": true, + "model_id": "anthropic.claude-haiku-4-5-20251001-v1:0", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-4-5.html (read 2026-11-07): bedrock-runtime Messages supported; Geo ids us./eu./au./jp./in.anthropic.claude-haiku-4-5-20251001-v1:0, Global id global.anthropic.claude-haiku-4-5-20251001-v1:0; no in-region form" } } } diff --git a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json index cf642646e..ba77539d3 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json @@ -3,7 +3,7 @@ "provider": "openai", "verified_at": "2026-09-24", "verified_region": "us-east-2", - "source": "Opus 5.5 cross-family target: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html (retrieved 2026-09-24). GPT-5.5 and GPT-5.4 rows re-read from their public model cards on 2026-11-06 (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html): both now list bedrock-runtime (Responses, Chat Completions, Converse via us./global. CRIS ids) and Chat Completions on bedrock-mantle. Other OpenAI entries retain their evidence from the superseded 2026-08-21 catalog (removed; see git history).", + "source": "Opus 5.5 cross-family target: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html (retrieved 2026-09-24). GPT-5.5 and GPT-5.4 rows re-read from their public model cards on 2026-11-06 (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html): both now list bedrock-runtime (Responses, Chat Completions, Converse via us./global. CRIS ids) and Chat Completions on bedrock-mantle. Other OpenAI entries retain their evidence from the superseded 2026-08-21 catalog (removed; see git history). The runtime_openai_responses and runtime_openai_chat path definitions and their per-model rows on every openai_gpt_* entry were added from the nine public model cards and the endpoints / inference-responses-api / inference-chat-completions pages on 2026-11-07; earlier rows and their evidence are unchanged.", "disclaimer": "The Opus cross-family target is updated to Opus 5.5; the GPT-5.5 and GPT-5.4 entries were re-read from their public model cards on 2026-11-06 (runtime CRIS paths, Chat Completions on mantle, 1.05M context, long-context pricing); the other same-model OpenAI paths and their earlier evidence remain unchanged. This is a migration target catalog, not an account-availability claim. Verify the chosen model, API, region and CRIS profile before implementation. Opus 5.5 uses CRIS on runtime outside eu-west-2, has 1M context/128K output, always-on adaptive thinking, and no Batch tier. Fable and Mythos remain excluded from default selection. Opus 5.5 is the only Opus model in the automatic selection pool; older Opus snapshots and pricing records are historical references. The runtime in-region exception in eu-west-2 was verified 2026-10-03; the original catalog and other model evidence dates remain unchanged.", "paths": { "mantle_openai_responses": { @@ -50,7 +50,7 @@ "supports_cris": true, "strengths": [ "Bedrock-native multi-model request surface", - "Supports Bedrock Guardrails (Converse API only), invocation logging, and CloudWatch", + "Supports Bedrock Guardrails (Converse API only) and model invocation logging — both bedrock-runtime-only; CloudWatch metrics exist on both endpoints (AWS/Bedrock on runtime, AWS/BedrockMantle on Mantle; https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-mantle-metrics.html, read 2026-11-07)", "This is a SAME-MODEL governance path via CRIS. GPT-5.6 Terra and Luna publish us./in./global. ids; us./global. only for GPT-5.6 Sol, GPT-5.5 and GPT-5.4 (their model cards list no in. profile; cards read 2026-11-07). The base URL for the OpenAI-compatible APIs is bedrock-runtime.{region}.amazonaws.com/openai/v1. The model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" ], "constraints": [ @@ -60,6 +60,53 @@ "Bedrock-native models generally require a Global or geography-scoped CRIS inference profile", "Availability must be probed in the target account and region" ] + }, + "runtime_openai_responses": { + "sdk": "openai", + "client": "OpenAI(base_url=bedrock-runtime)", + "endpoint": "https://bedrock-runtime.{region}.amazonaws.com/openai/v1", + "api": "responses", + "iam_action": "bedrock:InvokeModel", + "iam_auth": "bedrock_bearer_token_or_sigv4", + "model_id_strategy": "cris_inference_profile", + "supports_cris": true, + "strengths": [ + "Default endpoint per bedrock-endpoint-policy.md: AWS recommends bedrock-runtime for most new applications (https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html, read 2026-11-07)", + "Keeps the OpenAI SDK and the source Responses API — only base_url and api_key change (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, read 2026-11-07)", + "Model invocation logging (including the OpenAI-compatible APIs) and CloudWatch AWS/Bedrock metrics apply (https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html, read 2026-11-07)", + "SAME-MODEL path whose model ID is the CRIS inference profile: us. or global. ids; in. additionally for GPT-5.6 Terra and Luna only (model cards read 2026-11-07)" + ], + "constraints": [ + "Always synchronous: background=true is rejected with a 400 error; store keeps its default (https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html, read 2026-11-07)", + "Default project only — additional Projects are bedrock-mantle only (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, read 2026-11-07)", + "No Web Search or other server-side/pre-configured tools on bedrock-runtime; those are bedrock-mantle selectors per bedrock-endpoint-policy.md", + "Guardrails and application inference profiles are Converse-only on the GPT-5.6 Sol/Terra/Luna, GPT-5.5, GPT-5.4 and GPT-6 Astra cards; the GPT-6 Sol, GPT-6 Luna and GPT-6.1 Sol cards list wider Guardrails API scopes while application inference profiles stay InvokeModel/Converse-only — read the selected card (cards read 2026-11-07)", + "IAM needs bedrock:InvokeModel on the inference profile and on the account's default project (arn:aws:bedrock:{region}:{account-id}:project/default)", + "Availability must be probed in the target account and region" + ] + }, + "runtime_openai_chat": { + "sdk": "openai", + "client": "OpenAI(base_url=bedrock-runtime)", + "endpoint": "https://bedrock-runtime.{region}.amazonaws.com/openai/v1", + "api": "chat_completions", + "iam_action": "bedrock:InvokeModel", + "iam_auth": "bedrock_bearer_token_or_sigv4", + "model_id_strategy": "cris_inference_profile", + "supports_cris": true, + "strengths": [ + "Default endpoint per bedrock-endpoint-policy.md: AWS recommends bedrock-runtime for most new applications (https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html, read 2026-11-07)", + "Keeps the OpenAI SDK and the source Chat Completions API at /openai/v1/chat/completions — only base_url and api_key change (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions.html, read 2026-11-07)", + "Model invocation logging (including the OpenAI-compatible APIs) and CloudWatch AWS/Bedrock metrics apply (https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html, read 2026-11-07)", + "SAME-MODEL path whose model ID is the CRIS inference profile: us. or global. ids; in. additionally for GPT-5.6 Terra and Luna only (model cards read 2026-11-07)" + ], + "constraints": [ + "Always synchronous and default project only — additional Projects, background inference and server-side/pre-configured tools (including Web Search) are bedrock-mantle only (https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html, read 2026-11-07)", + "Guardrails and application inference profiles are Converse-only on the GPT-5.6 Sol/Terra/Luna, GPT-5.5, GPT-5.4 and GPT-6 Astra cards; the GPT-6 Sol, GPT-6 Luna and GPT-6.1 Sol cards list wider Guardrails API scopes while application inference profiles stay InvokeModel/Converse-only — read the selected card (cards read 2026-11-07)", + "Structured outputs and prompt caching on Chat Completions are model-specific: the GPT-5.5 and GPT-5.4 cards list both as Responses-only on bedrock-runtime; the GPT-5.6 and GPT-6 cards list Chat Completions for structured outputs and implicit caching (cards read 2026-11-07)", + "IAM needs bedrock:InvokeModel on the inference profile and on the account's default project (arn:aws:bedrock:{region}:{account-id}:project/default)", + "Availability must be probed in the target account and region" + ] } }, "models": { @@ -103,6 +150,18 @@ "model_id": "openai.gpt-5.6-sol", "requires_cris": true, "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x" + }, + "runtime_openai_responses": { + "available": true, + "model_id": "openai.gpt-5.6-sol", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-07): bedrock-runtime lists Responses supported (base URL bedrock-runtime.{region}.amazonaws.com/openai/v1); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol" + }, + "runtime_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.6-sol", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported (/openai/v1/chat/completions); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol" } } }, @@ -153,6 +212,18 @@ "model_id": "openai.gpt-5.5", "requires_cris": true, "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.5 (Geo) or global.openai.gpt-5.5 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs" + }, + "runtime_openai_responses": { + "available": true, + "model_id": "openai.gpt-5.5", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-07): bedrock-runtime lists Responses supported; structured outputs and implicit prompt caching are Responses-only on runtime; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" + }, + "runtime_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.5", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; the card scopes structured outputs and implicit prompt caching to Responses; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" } } }, @@ -193,6 +264,18 @@ "model_id": "openai.gpt-5.6-terra", "requires_cris": true, "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x" + }, + "runtime_openai_responses": { + "available": true, + "model_id": "openai.gpt-5.6-terra", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-07): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra" + }, + "runtime_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.6-terra", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra" } } }, @@ -233,6 +316,18 @@ "model_id": "openai.gpt-5.6-luna", "requires_cris": true, "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x" + }, + "runtime_openai_responses": { + "available": true, + "model_id": "openai.gpt-5.6-luna", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-07): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna" + }, + "runtime_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.6-luna", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna" } } }, @@ -286,6 +381,18 @@ "model_id": "openai.gpt-5.4", "requires_cris": true, "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.4 (Geo) or global.openai.gpt-5.4 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs" + }, + "runtime_openai_responses": { + "available": true, + "model_id": "openai.gpt-5.4", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-07): bedrock-runtime lists Responses supported; structured outputs and implicit prompt caching are Responses-only on runtime; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" + }, + "runtime_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.4", + "requires_cris": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; the card scopes structured outputs and implicit prompt caching to Responses; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" } } }, @@ -427,6 +534,126 @@ "structured_output_json" ], "evidence": "AWS model card, runtime JSON Schema reverified 2026-10-03: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html. Runtime APIs: Responses, Chat Completions, Converse; Invoke and Messages unsupported. US Geo/Global CRIS only, no in-region or EU/India Geo form. Guardrails and application inference profiles are Converse-only. Non-streaming JSON Schema outputs supported: Converse requires outputConfig.textFormat.structure.jsonSchema and additionalModelRequestFields.text.format.strict=true. Streaming structured output and server-side tool use are not evidenced. Runtime TPM accounting uses 10x output-token burndown." + }, + "runtime_openai_responses": { + "available": true, + "model_id": "openai.gpt-6-astra", + "requires_cris": true, + "supported_regions": [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "ca-central-1", + "eu-central-1", + "eu-north-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "sa-east-1" + ], + "inference_profile_regions": { + "us.openai.gpt-6-astra": [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "ca-central-1" + ], + "global.openai.gpt-6-astra": [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "ca-central-1", + "eu-central-1", + "eu-north-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "sa-east-1" + ] + }, + "capabilities": [ + "reasoning", + "image_input_vision", + "streaming", + "structured_output_json", + "prompt_caching" + ], + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html (read 2026-11-07): bedrock-runtime lists Responses supported; runtime features: response streaming, structured outputs (JSON Schema), implicit and explicit prompt caching on Responses, reasoning summaries; application inference profiles are InvokeModel/Converse-only and server-side tool use is unsupported; no in-region form; Geo id us.openai.gpt-6-astra, Global id global.openai.gpt-6-astra" + }, + "runtime_openai_chat": { + "available": true, + "model_id": "openai.gpt-6-astra", + "requires_cris": true, + "supported_regions": [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "ca-central-1", + "eu-central-1", + "eu-north-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "sa-east-1" + ], + "inference_profile_regions": { + "us.openai.gpt-6-astra": [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "ca-central-1" + ], + "global.openai.gpt-6-astra": [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "ca-central-1", + "eu-central-1", + "eu-north-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "sa-east-1" + ] + }, + "capabilities": [ + "reasoning", + "image_input_vision", + "streaming", + "structured_output_json", + "prompt_caching" + ], + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; runtime features: response streaming, structured outputs (JSON Schema), implicit and explicit prompt caching on Chat Completions; application inference profiles are InvokeModel/Converse-only and server-side tool use is unsupported; no in-region form; Geo id us.openai.gpt-6-astra, Global id global.openai.gpt-6-astra" } } }, @@ -452,7 +679,9 @@ "paths": { "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported."} + "runtime_converse": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported."}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-07): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"} } }, "openai_gpt_6_luna": { @@ -477,7 +706,9 @@ "paths": { "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke."} + "runtime_converse": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke."}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-07): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"} } }, "openai_gpt_6_1_sol": { @@ -502,7 +733,9 @@ "paths": { "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported."} + "runtime_converse": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported."}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-07): bedrock-runtime lists Responses supported; Guardrails are listed for Chat Completions/InvokeModel/Converse but not Responses, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; Guardrails listed on Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"} } }, "anthropic_claude_sonnet_5": { diff --git a/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py b/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py index 9b2633d5a..309474f16 100644 --- a/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py +++ b/plugins/aws-startup-advisor/tests/test_endpoint_policy_contract.py @@ -4,8 +4,8 @@ new applications and `bedrock-mantle` for capabilities available only there (https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html, read 2026-11-07). The plugin states that rule once, in -`skills/shared/ai/bedrock-endpoint-policy.md`, vendors it into gcp-to-aws and -azure-to-aws, and requires every `mantle_*` design row to record a +`skills/shared/ai/bedrock-endpoint-policy.md`, vendors it into gcp-to-aws, +azure-to-aws and agent-advisor, and requires every `mantle_*` design row to record a `mantle_selector`. These checks pin the policy file's contract and the design files and the llm-to-bedrock orchestrator that must point at it. """ @@ -20,6 +20,7 @@ POLICY_VENDORED = [ PLUGIN_ROOT / "skills" / "gcp-to-aws" / "references" / "vendored" / "ai" / "bedrock-endpoint-policy.md", PLUGIN_ROOT / "skills" / "azure-to-aws" / "references" / "vendored" / "ai" / "bedrock-endpoint-policy.md", + PLUGIN_ROOT / "skills" / "agent-advisor" / "references" / "vendored" / "ai" / "bedrock-endpoint-policy.md", ] ANTHROPIC_GUIDE = PLUGIN_ROOT / "skills" / "shared" / "ai" / "ai-anthropic-to-bedrock.md" From 16814de7fa308bd6cf4c6a9a4a2af786d8a2c2b4 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sat, 7 Nov 2026 21:45:46 -0800 Subject: [PATCH 28/41] fix(aws-startup-advisor): agent-advisor Anthropic recommendations default to bedrock-runtime Messages; Mantle only by mantle_selector anthropic_model_recommendation.py now follows skills/shared/ai/bedrock-endpoint-policy.md: - runtime_messages joins SUPPORTED_PATHS / ANTHROPIC_PATHS (the OpenAI runtime paths are accepted for preferred_api_path validation); MANTLE_SELECTORS carries the policy's 7-value enum. - _converse_required = guardrails in governance or multi_model_converse. invocation_logging / cloudwatch alone no longer force Converse; they are satisfied on every bedrock-runtime path. - _path_constraints: preferred -> native_required (runtime_invoke) -> converse_required (runtime_converse) -> mantle_selector (mantle_messages) -> preserve_messages_api / newest_anthropic_betas (runtime_messages) -> Messages source [runtime_messages, runtime_converse, runtime_invoke] -> default [runtime_converse, runtime_invoke, runtime_messages]. Conflicts: messages_vs_converse -> [runtime_messages, runtime path]; mantle_selector_vs_runtime -> [mantle_messages, runtime path]; preferred mantle_* without a selector -> mantle_selector_required (own [BLOCKS] finding plus the model_path_decision_required block, which now lists the conflict codes and names the two options). Unknown selector values raise ValueError. - recommend_anthropic_workload derives mantle_selector in_region_processing when data_residency == in_region_required and the runtime candidate is CRIS-only in the region (Opus 5.5 in eu-west-2 stays on runtime_messages with its bare id). Every result emits mantle_selector (string on mantle_messages, null otherwise); the Mantle decision option carries it. newest_anthropic_betas adds the [TUNE] beta_header_verification finding. - Wording: version_header / prompt_cache_shape deltas and _compatibility treat runtime_messages like Mantle Messages (prompt caching native on both). Tests: test_model_recommendation.py rewritten for the runtime default (selector parametrization, logging-only governance, decision_required option sets, in-Region derivation, fail-closed preferred Mantle, schema validation of default and decision_required outputs); test_opus55_integration.py and test_verify_model_path.py pass mantle_selector workspaces where they still probe Mantle and add runtime_messages coverage. Verification (worktree fix-agent-advisor-runtime-default, base f3c01556): - cd skills/agent-advisor/scripts && uv run --group dev pytest -q -> 629 passed (602 after FEAT-001; +27) - same, -v on test_model_recommendation.py test_opus55_integration.py test_verify_model_path.py -> 219 passed - git diff --check -> clean No catalog rows changed in this commit (no new model-card reads; FEAT-001 cards read 2026-11-07). Not run here: plugin-level mise gates and gitleaks (FEAT-004); the known pre-existing upstream gitleaks finding on f8d09fd1 is unrelated to this commit. --- .../scripts/anthropic_model_recommendation.py | 193 ++++++++++++++---- .../scripts/test_model_recommendation.py | 157 +++++++++++++- .../scripts/test_opus55_integration.py | 25 ++- .../scripts/test_verify_model_path.py | 34 ++- 4 files changed, 349 insertions(+), 60 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/anthropic_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/anthropic_model_recommendation.py index 54c9e48e3..5b02be7de 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/anthropic_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/anthropic_model_recommendation.py @@ -15,11 +15,26 @@ "mantle_openai_responses", "runtime_converse", "runtime_invoke", + "runtime_messages", + "runtime_openai_chat", + "runtime_openai_responses", } ANTHROPIC_PATHS = ( "mantle_messages", "runtime_converse", "runtime_invoke", + "runtime_messages", +) +# The only reasons to leave the default bedrock-runtime endpoint for bedrock-mantle +# (skills/shared/ai/bedrock-endpoint-policy.md). Nothing else selects Mantle. +MANTLE_SELECTORS = ( + "background_inference", + "server_side_tools", + "web_search", + "projects", + "workspaces", + "mantle_only_model_or_api", + "in_region_processing", ) # Sources this module can genuinely reason about. `none` and `unknown` belong here: a source with no # detected provider is treated as Bedrock-native and gets the Anthropic pool. Any OTHER provider @@ -209,16 +224,25 @@ def _required_capabilities(requirements): return capabilities +def _converse_required(requirements): + # Guardrails and the multi-model surface are Converse-only. invocation_logging and + # cloudwatch alone are satisfied by every bedrock-runtime path, including Messages. + return "guardrails" in requirements.get("governance", []) or requirements.get( + "multi_model_converse", False + ) + + def _path_constraints(workload): source = workload["source"] requirements = workload["requirements"] preferred = requirements.get("preferred_api_path") if preferred and preferred not in SUPPORTED_PATHS: raise ValueError(f"unsupported preferred_api_path: {preferred}") + selector = requirements.get("mantle_selector") + if selector and selector not in MANTLE_SELECTORS: + raise ValueError(f"unsupported mantle_selector: {selector}") - runtime_required = bool(requirements.get("governance")) or requirements.get( - "multi_model_converse", False - ) + converse_required = _converse_required(requirements) native_required = requirements.get("requires_native_payload", False) messages_required = requirements.get("preserve_messages_api", False) or requirements.get( "newest_anthropic_betas", False @@ -231,24 +255,41 @@ def _path_constraints(workload): runtime_path = "runtime_invoke" if native_required else "runtime_converse" conflicts = [] - if messages_required and (runtime_required or native_required): - conflicts.append("messages_vs_runtime") + option_paths = [] + if messages_required and (converse_required or native_required): + conflicts.append("messages_vs_converse") + option_paths = ["runtime_messages", runtime_path] + if selector and (converse_required or native_required): + conflicts.append("mantle_selector_vs_runtime") + option_paths = ["mantle_messages", runtime_path] if preferred: - if preferred.startswith("mantle_") and (runtime_required or native_required): + if preferred.startswith("mantle_") and (converse_required or native_required): conflicts.append("preferred_path_vs_runtime") - if preferred.startswith("runtime_") and messages_required: + option_paths = option_paths or ["mantle_messages", runtime_path] + if preferred in {"runtime_converse", "runtime_invoke"} and messages_required: conflicts.append("preferred_path_vs_messages") + option_paths = option_paths or ["runtime_messages", runtime_path] if native_required and preferred != "runtime_invoke": conflicts.append("preferred_path_vs_native_payload") + option_paths = option_paths or ["mantle_messages", runtime_path] + if preferred.startswith("mantle_") and not selector: + conflicts.append("mantle_selector_required") + option_paths = option_paths or [ + "runtime_messages" if source_messages or messages_required else runtime_path, + preferred, + ] if conflicts: return { "paths": [], "conflicts": sorted(set(conflicts)), - "option_paths": ["mantle_messages", runtime_path], + "option_paths": option_paths, "rationale": [ - "Messages continuity or beta requirements conflict with required Bedrock runtime capabilities." + "The recorded requirements cannot be satisfied on one Bedrock path: " + + ", ".join(sorted(set(conflicts))) + + "." ], + "mantle_selector": selector, } if preferred: return { @@ -256,6 +297,7 @@ def _path_constraints(workload): "conflicts": [], "option_paths": [], "rationale": [f"User selected the {preferred} API path."], + "mantle_selector": selector, } if native_required: return { @@ -263,39 +305,51 @@ def _path_constraints(workload): "conflicts": [], "option_paths": [], "rationale": ["InvokeModel is required for the provider-native Bedrock body."], + "mantle_selector": selector, } - if runtime_required: + if converse_required: return { "paths": ["runtime_converse"], "conflicts": [], "option_paths": [], - "rationale": [ - "Converse is required by governance, logging, or multi-model requirements." - ], + "rationale": ["Converse is required by Guardrails or multi-model requirements."], + "mantle_selector": selector, } - if messages_required: + if selector: return { "paths": ["mantle_messages"], "conflicts": [], "option_paths": [], "rationale": [ - "Mantle is required to preserve Messages semantics or newest beta features." + f"Mantle is selected by mantle_selector {selector} (bedrock-endpoint-policy.md)." ], + "mantle_selector": selector, + } + messages_rationale = ( + "bedrock-runtime Messages preserves the first-party Messages API on the default endpoint." + ) + if messages_required: + return { + "paths": ["runtime_messages"], + "conflicts": [], + "option_paths": [], + "rationale": [messages_rationale], + "mantle_selector": None, } if source_messages: return { - "paths": list(ANTHROPIC_PATHS), + "paths": ["runtime_messages", "runtime_converse", "runtime_invoke"], "conflicts": [], "option_paths": [], - "rationale": [ - "Mantle is preferred for an existing first-party Messages API call." - ], + "rationale": [messages_rationale], + "mantle_selector": None, } return { - "paths": ["runtime_converse", "runtime_invoke", "mantle_messages"], + "paths": ["runtime_converse", "runtime_invoke", "runtime_messages"], "conflicts": [], "option_paths": [], "rationale": ["Converse is the default for a new governance-ready Bedrock workload."], + "mantle_selector": None, } @@ -430,19 +484,33 @@ def _resolve_invocation_model_id(model_id, requires_cris, requirements, model=No def _decision_options(catalog, workload, option_paths, region=None): + requirements = workload["requirements"] + selector = requirements.get("mantle_selector") options = [] for path in dict.fromkeys(option_paths): - candidates = _build_candidates(catalog, [path], workload["requirements"], region) + candidates = _build_candidates(catalog, [path], requirements, region) if not candidates: continue - tradeoff = ( - "Preserves Messages code and beta access but gives up runtime-only governance." - if path == "mantle_messages" - else "Provides runtime governance but requires rewriting the Messages integration." - ) - options.append( - _candidate_summary(candidates[0], workload["requirements"], tradeoff, region) - ) + if path == "runtime_messages": + tradeoff = ( + "Keeps the Anthropic Messages API on bedrock-runtime (IAM, invocation logging, " + "CloudWatch); Converse-only features such as Guardrails and the multi-model " + "surface are not on this API." + ) + elif path == "mantle_messages": + tradeoff = ( + f"Mantle Messages selected by mantle_selector {selector}; gives up " + "bedrock-runtime-only governance (Guardrails, invocation logging)." + if selector + else "Mantle Messages requires a mantle_selector (bedrock-endpoint-policy.md); " + "gives up bedrock-runtime-only governance (Guardrails, invocation logging)." + ) + else: + tradeoff = "Provides runtime governance but requires rewriting the Messages integration." + option = _candidate_summary(candidates[0], requirements, tradeoff, region) + if path == "mantle_messages": + option["mantle_selector"] = selector + options.append(option) return options @@ -514,7 +582,8 @@ def _migration_deltas(source, source_analysis, path, feature_status, requirement _delta( "version_header", "path", - "Mantle uses the HTTP version header; runtime uses the Bedrock body field.", + "Messages (Mantle or runtime) sends the anthropic-version header through the SDK; " + "Converse and Invoke use the Bedrock body field.", ), _delta( "error_and_quota_surface", @@ -561,7 +630,8 @@ def _migration_deltas(source, source_analysis, path, feature_status, requirement _delta( "prompt_cache_shape", "path", - "Mantle cache_control and Converse cachePoint use different request shapes.", + "Messages cache_control (Mantle or runtime) and Converse cachePoint use " + "different request shapes.", ) ) return deltas @@ -576,11 +646,12 @@ def _compatibility(feature_status, requirements, path): for feature in detected if feature in {"citations", "streaming", "tool_use", "vision"} } - if "prompt_caching" in detected and path == "mantle_messages": + messages_path = path in {"mantle_messages", "runtime_messages"} + if "prompt_caching" in detected and messages_path: native.add("prompt_caching") portable = detected.intersection(_VERSION_SCAN_FEATURES) portable.update(detected.intersection({"models_api", "structured_output"})) - if "prompt_caching" in detected and path != "mantle_messages": + if "prompt_caching" in detected and not messages_path: portable.add("prompt_caching") rearchitecture = detected.intersection(_REARCHITECTURE_FEATURES) critical = set(requirements.get("critical_features", [])) @@ -728,15 +799,27 @@ def _decision_required(workload, region, catalog, constraints): "catalog has no candidate for every conflicting model/path option " f"for workload {workload['workload_id']}" ) + option_names = " or ".join(option["api_path"] for option in decision_options) blocks = [ _finding( "model_path_decision_required", "[BLOCKS]", - "Messages continuity conflicts with required Bedrock runtime capabilities.", - "Choose the Mantle continuity option or the runtime governance option, " - "update requirements, and rerun Model Recommend.", + "The recorded requirements cannot share one Bedrock path: " + + ", ".join(constraints["conflicts"]), + f"Choose {option_names}, update requirements, and rerun Model Recommend.", ) ] + if "mantle_selector_required" in constraints["conflicts"]: + blocks.append( + _finding( + "mantle_selector_required", + "[BLOCKS]", + "A bedrock-mantle path was preferred without a mantle_selector; nothing but a " + "selector from bedrock-endpoint-policy.md moves a workload off bedrock-runtime.", + "Record the applicable requirements.mantle_selector value or choose the " + "bedrock-runtime option, then rerun Model Recommend.", + ) + ) return { "workload_id": workload["workload_id"], "provider_module": "anthropic", @@ -752,6 +835,7 @@ def _decision_required(workload, region, catalog, constraints): "model_identity": None, "api_path": None, "invocation_model_id": None, + "mantle_selector": None, "decision_options": decision_options, "alternatives": [], "rationale": constraints["rationale"], @@ -796,6 +880,30 @@ def recommend_anthropic_workload(workload, region, catalog): model = chosen["model"] path = chosen["path"] path_config = chosen["path_config"] + requirements = workload["requirements"] + rationale = list(constraints["rationale"]) + mantle_selector = constraints["mantle_selector"] if path == "mantle_messages" else None + if ( + requirements.get("data_residency") == "in_region_required" + and path.startswith("runtime_") + and path_config["requires_cris"] + and not _runtime_in_region_allowed(path_config["model_id"], model, region, requirements) + and not (_converse_required(requirements) or requirements.get("requires_native_payload")) + ): + # Derived selector (bedrock-endpoint-policy.md in_region_processing): bedrock-runtime + # serves this model here only through a CRIS profile, which may route out of the Region. + in_region = _build_candidates(catalog, ["mantle_messages"], requirements, region) + if in_region: + chosen = in_region[0] + model = chosen["model"] + path = chosen["path"] + path_config = chosen["path_config"] + mantle_selector = "in_region_processing" + rationale.append( + "Strict in-Region processing was requested and bedrock-runtime serves this model " + "only through a CRIS profile here; Mantle Messages keeps processing in-Region " + "(mantle_selector in_region_processing)." + ) source_analysis = _source_analysis(workload["source"], model["version"]) feature_status = _feature_assessment( workload, @@ -846,7 +954,17 @@ def recommend_anthropic_workload(workload, region, catalog): "Opus 5.5 does not support Bedrock Batch. Use on-demand inference, " "or select a verified batch-capable alternative such as Opus 4.6." ) - rationale = list(constraints["rationale"]) + if requirements.get("newest_anthropic_betas"): + tuning.append( + _finding( + "beta_header_verification", + "[TUNE]", + "Newest Anthropic beta headers are model- and endpoint-specific.", + "Verify each beta header on the selected endpoint; if a beta is served on " + "bedrock-mantle only, record mantle_selector mantle_only_model_or_api and rerun " + "Model Recommend.", + ) + ) rationale.append( f"{model['display_name']} is the highest-ranked {chosen['driver']} model " f"that satisfies the {path} constraints." @@ -880,6 +998,7 @@ def recommend_anthropic_workload(workload, region, catalog): }, "api_path": path, "invocation_model_id": invocation_model_id, + "mantle_selector": mantle_selector, "decision_options": [], "alternatives": alternatives, "rationale": rationale, diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_model_recommendation.py index 58274904c..5ffb47166 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_model_recommendation.py @@ -62,27 +62,59 @@ def _codes(items): return {item["code"] for item in items} -def test_default_anthropic_messages_prefers_mantle_and_sonnet5(): - # Sonnet 5 (launched 2026-06-30) supports Mantle Messages, so the balanced - # priority no longer has to escalate to Opus for Messages continuity. - rec = _recommend() +def _schema(name): + scripts = pathlib.Path(model_recommendation.__file__).parent + return json.loads((scripts / "schemas" / name).read_text()) + + +def test_default_anthropic_messages_lands_on_runtime_messages(): + # bedrock-runtime is the default endpoint (bedrock-endpoint-policy.md); a Messages + # source keeps its API on bedrock-runtime Messages. Nothing selects Mantle here. + result = model_recommendation.recommend(_input()) + rec = result["workloads"]["support-agent"] assert rec["decision_status"] == "recommended" assert rec["primary_model"] == "anthropic.claude-sonnet-5" assert rec["model_identity"]["display_name"] == "Claude Sonnet 5" assert rec["model_identity"]["version"] == "5.0" - assert rec["api_path"] == "mantle_messages" - assert rec["invocation_model_id"] == "anthropic.claude-sonnet-5" + assert rec["api_path"] == "runtime_messages" + assert rec["invocation_model_id"] is None # residency unknown: no CRIS prefix guessed + assert rec["mantle_selector"] is None + assert rec["model_identity"]["requires_cris"] is True assert rec["model_identity"]["path_model_id"] == rec["primary_model"] assert rec["verification"]["probe_status"] == "not_run" + assert not any("mantle" in block["message"].lower() for block in rec["blocks"]) + jsonschema.validate(result, _schema("model-recommendation.json")) -def test_cost_priority_on_messages_uses_clean_mantle_haiku_id(): - rec = _recommend(_workload(requirements={"priority": "cost"})) +def test_cost_priority_with_workspaces_selector_uses_clean_mantle_haiku_id(): + rec = _recommend( + _workload(requirements={"priority": "cost", "mantle_selector": "workspaces"}) + ) assert rec["api_path"] == "mantle_messages" assert rec["primary_model"] == "anthropic.claude-haiku-4-5" + assert rec["invocation_model_id"] == "anthropic.claude-haiku-4-5" assert rec["model_identity"]["requires_cris"] is False + assert rec["mantle_selector"] == "workspaces" + + +@pytest.mark.parametrize("selector", anthropic_model_recommendation.MANTLE_SELECTORS) +def test_every_mantle_selector_selects_mantle_messages_and_is_echoed(selector): + rec = _recommend(_workload(requirements={"mantle_selector": selector})) + + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "mantle_messages" + assert rec["mantle_selector"] == selector + assert any(f"mantle_selector {selector}" in line for line in rec["rationale"]) + + +def test_mantle_selectors_match_the_input_schema_enum(): + enum = _schema("model-recommendation-input.json")["properties"]["workloads"]["items"][ + "properties"]["requirements"]["properties"]["mantle_selector"]["enum"] + assert list(anthropic_model_recommendation.MANTLE_SELECTORS) == enum + with pytest.raises(ValueError, match="unsupported mantle_selector"): + _recommend(_workload(requirements={"mantle_selector": "because"})) def test_governance_selects_runtime_converse_when_messages_is_a_preference(): @@ -95,6 +127,21 @@ def test_governance_selects_runtime_converse_when_messages_is_a_preference(): assert rec["decision_status"] == "recommended" assert rec["api_path"] == "runtime_converse" assert rec["primary_model"] == "anthropic.claude-sonnet-5" + assert rec["mantle_selector"] is None + + +@pytest.mark.parametrize("governance", [["invocation_logging"], ["cloudwatch"], + ["invocation_logging", "cloudwatch"]]) +def test_runtime_only_logging_governance_keeps_required_messages_on_runtime(governance): + # invocation logging and CloudWatch apply to every bedrock-runtime path, so they + # never force a Messages workload onto Converse. + rec = _recommend( + _workload(requirements={"preserve_messages_api": True, "governance": governance}) + ) + + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "runtime_messages" + assert rec["mantle_selector"] is None def test_required_messages_and_governance_require_user_decision(): @@ -110,11 +157,99 @@ def test_required_messages_and_governance_require_user_decision(): assert rec["decision_status"] == "decision_required" assert rec["primary_model"] is None assert rec["api_path"] is None + assert rec["mantle_selector"] is None assert {option["api_path"] for option in rec["decision_options"]} == { - "mantle_messages", + "runtime_messages", "runtime_converse", } - assert "model_path_decision_required" in _codes(rec["blocks"]) + assert all(option.get("mantle_selector") is None for option in rec["decision_options"]) + block = next(item for item in rec["blocks"] if item["code"] == "model_path_decision_required") + assert "messages_vs_converse" in block["message"] + assert "runtime_messages" in block["remediation"] and "runtime_converse" in block["remediation"] + jsonschema.validate( + model_recommendation.recommend(_input(_workload(requirements={ + "preserve_messages_api": True, "governance": ["guardrails"]}))), + _schema("model-recommendation.json"), + ) + + +def test_selector_and_guardrails_require_user_decision_with_mantle_option(): + rec = _recommend( + _workload(requirements={"mantle_selector": "web_search", "governance": ["guardrails"]}) + ) + + assert rec["decision_status"] == "decision_required" + options = {option["api_path"]: option for option in rec["decision_options"]} + assert set(options) == {"mantle_messages", "runtime_converse"} + assert options["mantle_messages"]["mantle_selector"] == "web_search" + assert "mantle_selector" not in options["runtime_converse"] + block = next(item for item in rec["blocks"] if item["code"] == "model_path_decision_required") + assert "mantle_selector_vs_runtime" in block["message"] + + +def test_in_region_required_on_a_cris_only_runtime_candidate_derives_mantle(): + rec = _recommend(_workload(requirements={"data_residency": "in_region_required"})) + + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "mantle_messages" + assert rec["mantle_selector"] == "in_region_processing" + assert rec["primary_model"] == "anthropic.claude-sonnet-5" + assert rec["invocation_model_id"] == "anthropic.claude-sonnet-5" + assert any("in_region_processing" in line for line in rec["rationale"]) + + +def test_in_region_required_keeps_opus55_london_on_runtime_messages(): + workload = _workload( + requirements={"priority": "quality", "data_residency": "in_region_required"} + ) + data = _input(workload) + data["region"] = "eu-west-2" + rec = model_recommendation.recommend(data)["workloads"]["support-agent"] + + assert rec["api_path"] == "runtime_messages" + assert rec["primary_model"] == "anthropic.claude-opus-5-5" + assert rec["invocation_model_id"] == "anthropic.claude-opus-5-5" + assert rec["model_identity"]["requires_cris"] is False + assert rec["mantle_selector"] is None + + +def test_preferred_mantle_path_without_selector_fails_closed(): + rec = _recommend(_workload(requirements={"preferred_api_path": "mantle_messages"})) + + assert rec["decision_status"] == "decision_required" + assert "mantle_selector_required" in _codes(rec["blocks"]) + options = {option["api_path"] for option in rec["decision_options"]} + assert options == {"runtime_messages", "mantle_messages"} + + +def test_preferred_mantle_path_with_selector_is_honored(): + rec = _recommend( + _workload(requirements={"preferred_api_path": "mantle_messages", + "mantle_selector": "projects"}) + ) + + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "mantle_messages" + assert rec["mantle_selector"] == "projects" + + +def test_preferred_runtime_messages_is_honored_and_does_not_conflict_with_required_messages(): + rec = _recommend( + _workload(requirements={"preferred_api_path": "runtime_messages", + "preserve_messages_api": True}) + ) + + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "runtime_messages" + + +def test_newest_betas_stay_on_runtime_messages_with_a_verification_tune(): + rec = _recommend(_workload(requirements={"newest_anthropic_betas": True})) + + assert rec["api_path"] == "runtime_messages" + assert rec["mantle_selector"] is None + finding = next(item for item in rec["tuning"] if item["code"] == "beta_header_verification") + assert "mantle_only_model_or_api" in finding["remediation"] def test_conflicting_paths_without_catalog_candidates_are_rejected(): @@ -224,7 +359,7 @@ def test_detected_version_features_emit_blocks_and_tuning(): @pytest.mark.parametrize( "requirements,expected_path", [ - ({"preserve_messages_api": True}, "mantle_messages"), + ({"preserve_messages_api": True}, "runtime_messages"), ({"governance": ["guardrails"]}, "runtime_converse"), ], ) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_opus55_integration.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_opus55_integration.py index 5403b8357..c47d5587a 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_opus55_integration.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_opus55_integration.py @@ -49,9 +49,20 @@ def _script(name): return module -@pytest.mark.parametrize("path", ["runtime_converse", "runtime_invoke", "mantle_messages"]) +ANTHROPIC_PATHS = ["runtime_converse", "runtime_invoke", "runtime_messages", "mantle_messages"] + + +def _preferred(path): + # A preferred Mantle path needs a mantle_selector; nothing else moves a workload off runtime. + requirements = {"preferred_api_path": path} + if path == "mantle_messages": + requirements["mantle_selector"] = "workspaces" + return requirements + + +@pytest.mark.parametrize("path", ANTHROPIC_PATHS) def test_quality_and_thinking_select_opus55_on_each_supported_path(path): - rec = _recommend(preferred_api_path=path, thinking_enabled=True, min_context_tokens=1000000) + rec = _recommend(thinking_enabled=True, min_context_tokens=1000000, **_preferred(path)) assert rec["primary_model"] == MODEL assert rec["model_identity"]["output_token_ceiling"] == 128000 assert rec["invocation_model_id"] == (MODEL if path == "mantle_messages" else "global." + MODEL) @@ -84,7 +95,7 @@ def test_invalid_or_forbidden_profiles_remain_blocked(region, requirements): def test_mantle_unavailable_region_does_not_restore_opus48(): - rec = _recommend("eu-west-1", preferred_api_path="mantle_messages") + rec = _recommend("eu-west-1", **_preferred("mantle_messages")) assert rec["primary_model"] == "anthropic.claude-sonnet-5" assert all("opus-4-8" not in item["model"] for item in rec["alternatives"]) @@ -179,10 +190,10 @@ def test_current_catalogs_have_no_opus48_migration_target(): assert "anthropic_claude_opus_4_8" not in oai._CONVERSE_TIER_ORDER -@pytest.mark.parametrize("path", ["runtime_converse", "runtime_invoke", "mantle_messages"]) +@pytest.mark.parametrize("path", ANTHROPIC_PATHS) @pytest.mark.parametrize("feature", ["structured_output", "assistant_prefill"]) def test_opus55_structured_remedies_never_prescribe_forced_tool_choice(path, feature): - rec = _recommend(preferred_api_path=path, detected_features=[feature]) + rec = _recommend(detected_features=[feature], **_preferred(path)) assert rec["primary_model"] == MODEL code = "assistant_prefill_removed" if feature == "assistant_prefill" else "structured_output_portable_pattern" remedy = next(item["remediation"] for item in rec["blocks"] if item["code"] == code) @@ -209,9 +220,9 @@ def test_price_cache_preserves_existing_deployment_rates_without_recommending_op @pytest.mark.parametrize("priority", ["quality", "balanced", "speed", "cost", "unknown"]) -@pytest.mark.parametrize("path", ["runtime_converse", "runtime_invoke", "mantle_messages"]) +@pytest.mark.parametrize("path", ANTHROPIC_PATHS) def test_automatic_primary_and_alternatives_never_select_opus48(priority, path): - rec = _recommend(priority=priority, preferred_api_path=path) + rec = _recommend(priority=priority, **_preferred(path)) targets = [rec["primary_model"], *(item["model"] for item in rec["alternatives"])] assert all("opus-4-8" not in target for target in targets) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py index 4d909e28b..599889f05 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py @@ -80,17 +80,40 @@ def _verify(requirements=None, **kwargs): ) +# Mantle is probed only when a mantle_selector put the workload there. +MANTLE_SELECTED = {"mantle_selector": "workspaces"} + + def test_mantle_probe_uses_recommended_clean_id(): client = FakeMantleClient() - result = _verify(mantle_client_factory=lambda region: client) + result = _verify(MANTLE_SELECTED, mantle_client_factory=lambda region: client) verification = result["workloads"]["support-agent"] assert verification["status"] == "passed" + assert verification["api_path"] == "mantle_messages" assert verification["response_model_id"] == "anthropic.claude-sonnet-5" assert client.messages.calls[0]["model"] == "anthropic.claude-sonnet-5" assert client.messages.calls[0]["max_tokens"] == 8 +def test_default_messages_probe_uses_runtime_messages_with_resolved_cris(): + messages = FakeMessages(response={"model": "global.anthropic.claude-sonnet-5"}) + client = FakeMantleClient(messages) + mantle_calls = [] + result = _verify( + {"data_residency": "global_allowed"}, + mantle_client_factory=lambda region: mantle_calls.append(region), + runtime_messages_client_factory=lambda region: client, + ) + verification = result["workloads"]["support-agent"] + + assert verification["status"] == "passed" + assert verification["api_path"] == "runtime_messages" + assert verification["response_model_id"] == "global.anthropic.claude-sonnet-5" + assert messages.calls[0]["model"] == "global.anthropic.claude-sonnet-5" + assert mantle_calls == [] + + def test_converse_probe_uses_resolved_cris_without_substitution(): client = FakeRuntimeClient() result = _verify( @@ -141,7 +164,7 @@ def test_unresolved_runtime_cris_does_not_create_client(): def test_probe_failure_is_structured(): client = FakeMantleClient(FakeMessages(error=PermissionError("denied"))) - result = _verify(mantle_client_factory=lambda region: client) + result = _verify(MANTLE_SELECTED, mantle_client_factory=lambda region: client) verification = result["workloads"]["support-agent"] assert verification["status"] == "failed" @@ -163,7 +186,7 @@ def test_decision_required_is_not_probed(): def test_verification_output_matches_schema(): - result = _verify(mantle_client_factory=lambda region: FakeMantleClient()) + result = _verify(MANTLE_SELECTED, mantle_client_factory=lambda region: FakeMantleClient()) schema = json.loads( ( pathlib.Path(verify_model_path.__file__).parent @@ -685,8 +708,9 @@ def factory(region): # --- bedrock-runtime Responses / Chat Completions / Messages probes ----------- # -# The engines do not yet emit these paths, so the recommendations are built by -# hand from a recommended result and re-pointed at the Runtime path. +# The OpenAI engine does not yet emit its Runtime paths, so these recommendations are +# built by hand from a recommended result and re-pointed at the path under test; this +# keeps every apiPath enum value covered independently of engine defaults. RECOMMENDATION_SCHEMA = json.loads( (pathlib.Path(verify_model_path.__file__).parent / "schemas" / "model-recommendation.json").read_text() From 64a029dcf60cefe8b4d240d22569a965aa478a34 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 07:58:49 -0800 Subject: [PATCH 29/41] fix(aws-startup-advisor): agent-advisor OpenAI recommendations default to bedrock-runtime Responses/Chat; Mantle only by mantle_selector; Web Search is a Mantle selector; same-family newer-first fallback PR 3 of 3, FEAT-003 (plan item 6). Engine: skills/agent-advisor/scripts/ openai_model_recommendation.py now follows skills/shared/ai/bedrock-endpoint-policy.md. - Default path: runtime_openai_responses / runtime_openai_chat on the same model (CRIS id resolved as runtime_converse does today). api_continuity alone never selects Mantle. - mantle_selector (MANTLE_SELECTORS imported from anthropic_model_recommendation): explicit requirements.mantle_selector, derived web_search (detected/critical feature), derived in_region_processing (data_residency in_region_required on a CRIS-only runtime candidate, no governance). Every result emits mantle_selector (string on mantle_* paths, null otherwise); decision options carry it too. - Governance split: _converse_required (guardrails / multi_model_converse) -> runtime_converse; invocation_logging / cloudwatch alone stay on the runtime OpenAI API. Conflicts: selector + runtime governance -> [Mantle, runtime_converse or runtime_openai_*]; continuity + Converse-only governance -> [runtime_openai_, runtime_converse] (no Mantle option without a selector). - _SOURCE_MODEL_KEYS += gpt-5.5 / gpt-5.4 (same-model identities). - _openai_candidate_order replaces _mantle_candidate_order: same model, then same GPT major family newest-first, then other OpenAI models newest-first (fixes the GPT-6 Sol Chat -> GPT-5.4 ordering PR 1 disclosed). - _HOSTED_TOOL_IMPACTS[web_search]: Mantle-only server-side tool, cites tool-use-server-side.html and web-search.html (read 2026-11-07); portable on mantle_* paths, rearchitecture on runtime paths. - preferred_api_path honored for every source (Astra block generalized); mantle_* preference without a selector -> preferred_api_path_conflict. - cris_residency_unresolved message branches on data_residency unknown; reshape blocks and feature findings cover both endpoints; uses_n keys on any Chat path. - schemas/model-recommendation.json: mantle_selector required on recommendation; allOf conditional api_path ^mantle_ => mantle_selector string. - Tests: test_openai_model_recommendation.py rewritten (five PR 1 "deliberate consequence" tests deleted and replaced); test_verify_model_path.py OpenAI fixtures use gpt-4 for Claude-tier cases, a selector for Mantle probes, and gain runtime siblings plus a schema-rejection test. No catalog, vendored, doc or llm-to-bedrock file touched; no new catalog rows (FEAT-001 model-card reads of 2026-11-07 stand). Verification (worktree P=plugins/aws-startup-advisor): - cd P/skills/agent-advisor/scripts && uv run --group dev pytest -q -> 694 passed (629 after FEAT-002; baseline 570 on f3c01556) - cd P/skills/llm-to-bedrock/scripts && uv run --group dev pytest -q -> 622 passed (unchanged) - cd P && uv run --no-project --with pytest python -m pytest tests/test_gpt54_55_cris_prefix_scope.py tests/test_endpoint_policy_contract.py -q -> 11 passed - git diff --check -> clean - Not run here (FEAT-004 full gates): mise run test, lint:* gates, markdownlint, root tools/validate.py. gitleaks is not installed locally; the known pre-existing upstream finding on f8d09fd1 is unrelated to this commit and is to be reported distinctly in the PR body. --- .../scripts/openai_model_recommendation.py | 518 +++++++++---- .../scripts/schemas/model-recommendation.json | 21 +- .../test_openai_model_recommendation.py | 698 ++++++++++++++---- .../scripts/test_verify_model_path.py | 102 ++- 4 files changed, 1034 insertions(+), 305 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py index 47aeed856..b64bd1fa7 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py @@ -1,15 +1,19 @@ """Deterministic OpenAI-to-Bedrock model/path recommendation for agent-advisor. -Factual baseline: an internal OpenAI-to-Bedrock migration reference (2026-06 live -probes and its recorded feature-status notes). The engine is offline: -no openai/boto3/token-generator imports at module load. Rankings are only produced -where the reference provides evidence; otherwise the engine returns an explicit -decision or a provisional limitation rather than inventing a comparative order. +Factual baseline: the public Bedrock model cards (docs.aws.amazon.com/bedrock/latest/ +userguide/model-card-openai-*.html, cited per catalog row with a read date) and the +shared Bedrock endpoint policy (skills/shared/ai/bedrock-endpoint-policy.md: +bedrock-runtime is the default endpoint; bedrock-mantle is selected only by a +mantle_selector). The engine is offline: no openai/boto3/token-generator imports at +module load. Rankings are only produced where the catalog provides evidence; +otherwise the engine returns an explicit decision or a provisional limitation rather +than inventing a comparative order. """ import re import anthropic_model_recommendation +from anthropic_model_recommendation import MANTLE_SELECTORS # --- Source model-family detection (ported from the reference compatibility matrix) --- # Reasoning generation: gpt-5 / gpt-6 variants and o1/o3/o4. @@ -79,10 +83,13 @@ def _delta(code, category, description): # --- Feature vocabulary (provider-scoped; codes match scanner feature-catalog) --- _HOSTED_TOOL_IMPACTS = { "web_search": ( - "Hosted web search does not execute live on Bedrock Mantle (verified passthrough: " - "no query issued, no citations).", - "Implement a client-side tool loop, or expose a server-side tool (MCP/Lambda) through " - "Bedrock AgentCore Gateway; use Knowledge Bases for document retrieval.", + "The built-in Web Search tool is a pre-configured server-side tool served only on " + "bedrock-mantle; bedrock-runtime does not offer it. Model support is per model card " + "(https://docs.aws.amazon.com/bedrock/latest/userguide/tool-use-server-side.html, " + "https://docs.aws.amazon.com/bedrock/latest/userguide/web-search.html, read 2026-11-07).", + "Keep the workload on bedrock-mantle with mantle_selector web_search and verify the " + "tool on the selected model, or replace it with a client-side search tool / Bedrock " + "AgentCore Gateway tool on bedrock-runtime.", ), "file_search_retrieval": ( "Hosted file search / retrieval is not a drop-in on Bedrock.", @@ -115,11 +122,9 @@ def _delta(code, category, description): } -# Converse tier map: a governance workload keeps its capability tier when moving -# from a GPT-5.x source the engine does not recognize as a same-model runtime -# identity (see _SOURCE_MODEL_KEYS) to Claude on runtime Converse. This is an -# engine recognition gap, not an endpoint fact: GPT-5.5 and GPT-5.4 do carry a -# cataloged runtime_converse path via us./global. CRIS ids. +# Converse tier map: the cross-family fallback behind the same-model candidate on +# runtime Converse. A governance workload whose source has no cataloged same-model +# identity (see _SOURCE_MODEL_KEYS) keeps its capability tier when moving to Claude. # GPT-5.6 Sol (frontier) -> Claude Opus 5.5 # GPT-5.6 Terra / 5.5 / 5.4 -> Claude Sonnet 5 (also the default tier) # GPT-5.6 Luna (fast/low-cost)-> Claude Haiku 4.5 @@ -145,6 +150,8 @@ def _converse_tier_for_source(source): "gpt-5.6-sol": "openai_gpt_5_6_sol", "gpt-5.6-terra": "openai_gpt_5_6_terra", "gpt-5.6-luna": "openai_gpt_5_6_luna", + "gpt-5.5": "openai_gpt_5_5", + "gpt-5.4": "openai_gpt_5_4", "gpt-6-astra": "openai_gpt_6_astra", "gpt-6-sol": "openai_gpt_6_sol", "gpt-6-luna": "openai_gpt_6_luna", @@ -182,21 +189,81 @@ def _converse_candidate_order(source, requirements=None): def _mantle_path(source, requirements): - same = _same_model_catalog_key(source) + """Mantle path preserving the source API: Chat for a Chat source whose API + continuity is required/preferred/unknown (the candidate walk skips models whose + Mantle Chat row is unavailable), Responses otherwise.""" if ( source.get("api_surface") == "chat_completions" - and _api_continuity(requirements) in {"required", "preferred"} - and same is not None - and same.startswith("openai_gpt_6") + and _api_continuity(requirements) in {"required", "preferred", "unknown"} ): return "mantle_openai_chat" return "mantle_openai_responses" -def _mantle_candidate_order(catalog, source): +def _runtime_openai_path(surface): + """bedrock-runtime path that preserves the source OpenAI API surface.""" + return "runtime_openai_chat" if surface == "chat_completions" else "runtime_openai_responses" + + +def _api_label(path): + return "Chat Completions" if path.endswith("_chat") else "Responses" + + +def _converse_required(requirements): + """Converse is the only API every card evidences for Guardrails / application + inference profiles; multi-model Converse is Converse by definition.""" + return "guardrails" in (requirements.get("governance") or []) or bool( + requirements.get("multi_model_converse", False) + ) + + +def _mantle_selector(workload, feature_assessment): + """Explicit requirements.mantle_selector, else the derived web_search selector. + in_region_processing is derived later, once the runtime candidate is known.""" + requirements = workload["requirements"] + selector = requirements.get("mantle_selector") + if selector is not None: + if selector not in MANTLE_SELECTORS: + raise ValueError(f"unsupported mantle_selector: {selector}") + return selector + if ( + feature_assessment.get("web_search") == "detected" + or "web_search" in (requirements.get("critical_features") or []) + ): + return "web_search" + return None + + +def _openai_candidate_order(catalog, source): + """[same model] + OpenAI-generation models, same family as the source's GPT major + first, then version descending, then catalog order. Sources with no parseable + GPT major keep catalog order after the same model. Claude entries never enter an + OpenAI-API path (they have no such path).""" same = _same_model_catalog_key(source) - keys = list(catalog["models"]) - return ([same] if same else []) + [key for key in keys if key != same] + version = _openai_version(_primary_source_id(source)) + try: + major = int(version.split(".")[0]) if version else None + except ValueError: + major = None + openai_keys = [ + key for key, model in catalog["models"].items() + if model.get("generation") != "bedrock_native" and key != same + ] + if major is not None: + index = {key: position for position, key in enumerate(openai_keys)} + + def rank(key): + model = catalog["models"][key] + parsed = _version_tuple(model.get("version")) or (0, 0) + return ( + 0 if model.get("family") == f"openai_gpt_{major}" else 1, + -parsed[0], + -parsed[1], + index[key], + ) + + openai_keys = sorted(openai_keys, key=rank) + return ([same] if same else []) + openai_keys def _hard_profile_requirement(requirements): @@ -239,7 +306,7 @@ def _catalog_model_for_path(catalog, path, detected_features=None, requirements= if region not in supported_regions: continue if ( - path == "runtime_converse" + path.startswith("runtime_") and not path_config.get("inference_profile_regions") and path_config.get("requires_cris") and _hard_profile_requirement(requirements or {}) @@ -548,8 +615,8 @@ def _feature_findings(detected_features, requirements, path=None, target_model=N Bedrock-native Converse target does not.""" detected = set(detected_features or []) detected.update(requirements.get("critical_features") or []) - is_responses = path == "mantle_openai_responses" - is_chat = path == "mantle_openai_chat" + is_responses = path in {"mantle_openai_responses", "runtime_openai_responses"} + is_chat = path in {"mantle_openai_chat", "runtime_openai_chat"} blocks, tuning, deltas, impacts = [], [], [], [] if "structured_output_json" in detected: @@ -566,6 +633,17 @@ def _feature_findings(detected_features, requirements, path=None, target_model=N "validate the schema and check refusal/incomplete responses before parsing.", ) ) + elif path == "runtime_openai_responses": + deltas.append( + _delta( + "structured_output_text_format", + "feature", + "On bedrock-runtime Responses, keep typed structured output: text.format " + "json_schema (strict) is documented on the selected model's card; " + "client.responses.parse(..., text_format=Model) builds that request — " + "verify in the target account.", + ) + ) elif is_responses: # Verified on Mantle: responses.parse(text_format=Model) returns a typed object. deltas.append( @@ -588,7 +666,7 @@ def _feature_findings(detected_features, requirements, path=None, target_model=N ) ) if requirements.get("uses_n") or "multiple_candidates_n" in detected: - if path == "mantle_openai_chat": + if is_chat: if target_model and target_model.get("display_name") == "OpenAI GPT-6 Astra (on Bedrock)": tuning.append( _finding( @@ -610,7 +688,7 @@ def _feature_findings(detected_features, requirements, path=None, target_model=N if "tool_or_function_calling" in detected: if is_chat: tool_description = ( - "On Mantle Chat Completions, preserve assistant tool_calls and continue each " + "On Chat Completions, preserve assistant tool_calls and continue each " "result as a role: tool message with the matching tool_call_id; the " "application executes the tool." ) @@ -631,6 +709,16 @@ def _feature_findings(detected_features, requirements, path=None, target_model=N "until those requirements are confirmed.", ) ) + elif path == "runtime_openai_responses": + deltas.append( + _delta( + "conversation_state_ownership", + "path", + "Verify store=True / previous_response_id continuation on bedrock-runtime " + "Responses in the target account before relying on it; application-owned " + "replay (store=False) is the portable default.", + ) + ) elif is_responses: deltas.append( _delta( @@ -647,7 +735,7 @@ def _feature_findings(detected_features, requirements, path=None, target_model=N _delta( "conversation_state_ownership", "path", - "Mantle Chat Completions uses application-owned message history; replay " + "Chat Completions uses application-owned message history; replay " "the prior user, assistant, tool-call, and tool-result messages on each " "request.", ) @@ -767,8 +855,14 @@ def _compatibility(detected_features, requirements, path, target_model=None): catalog_caps = set((target_model or {}).get("capabilities") or []) native = sorted(text_features.intersection(catalog_caps)) if target_model else [] unsupported = sorted(text_features - catalog_caps) if target_model else [] - rearchitecture = sorted(detected.intersection(_HOSTED_TOOL_IMPACTS)) - portable = sorted(detected.intersection(_PORTABLE_FEATURES)) + hosted = detected.intersection(_HOSTED_TOOL_IMPACTS) + portable_features = detected.intersection(_PORTABLE_FEATURES) + if (path or "").startswith("mantle_") and "web_search" in hosted: + # Web Search is a Mantle server-side tool: on a Mantle path it ports. + hosted.discard("web_search") + portable_features.add("web_search") + rearchitecture = sorted(hosted) + portable = sorted(portable_features) return { "native": native, "portable": portable, @@ -842,47 +936,67 @@ def _verification(region, catalog, path, requires_cris, invocation_model_id, sel } -def _decision_options(catalog, workload, region): - """Two-sided option set: the selected Mantle API vs runtime Converse governance.""" +def _decision_options(catalog, workload, region, selector=None): + """Two-sided option set. Option A preserves the OpenAI SDK and the source API: + the Mantle path when a mantle_selector exists, else the bedrock-runtime + OpenAI path. Option B is runtime Converse governance.""" options = [] + source = workload["source"] + requirements = workload["requirements"] detected = [feature for feature, status in _feature_assessment(workload).items() if status == "detected"] - astra = _same_model_runtime_key(workload["source"]) == "openai_gpt_6_astra" - mantle_path = _mantle_path(workload["source"], workload["requirements"]) - if astra: - preferred = workload["requirements"].get("preferred_api_path") - if preferred in {"mantle_openai_chat", "mantle_openai_responses"}: - mantle_path = preferred - elif workload["source"].get("api_surface") == "chat_completions": - mantle_path = "mantle_openai_chat" - # A decision may offer changing an incompatible Chat preference to the - # documented Responses caching path; it must not silently select it. - needed = set(detected) | set(workload["requirements"].get("critical_features") or []) - if "prompt_caching" in needed: - mantle_path = _mantle_path(workload["source"], workload["requirements"]) - mantle, _ = _catalog_model_for_path( - catalog, mantle_path, detected, workload["requirements"], - candidate_order=["openai_gpt_6_astra"] if astra else _mantle_candidate_order(catalog, workload["source"]), - region=region, + astra = _same_model_runtime_key(source) == "openai_gpt_6_astra" + surface = source.get("api_surface") + if selector: + openai_path = _mantle_path(source, requirements) + if astra: + preferred = requirements.get("preferred_api_path") + if preferred in {"mantle_openai_chat", "mantle_openai_responses"}: + openai_path = preferred + elif surface == "chat_completions": + openai_path = "mantle_openai_chat" + # A decision may offer changing an incompatible Chat preference to the + # documented Mantle Responses caching path; it must not silently select it. + needed = set(detected) | set(requirements.get("critical_features") or []) + if "prompt_caching" in needed: + openai_path = "mantle_openai_responses" + else: + openai_path = _runtime_openai_path(surface) + candidate_order = ( + ["openai_gpt_6_astra"] if astra else _openai_candidate_order(catalog, source) ) - mantle_requires_region_change = False - if mantle is None and region != "unknown": - mantle, _ = _catalog_model_for_path( - catalog, mantle_path, detected, workload["requirements"], - candidate_order=_mantle_candidate_order(catalog, workload["source"]), + openai_hit, _ = _catalog_model_for_path( + catalog, openai_path, detected, requirements, + candidate_order=candidate_order, region=region, + ) + requires_region_change = False + if openai_hit is None and selector and region != "unknown": + openai_hit, _ = _catalog_model_for_path( + catalog, openai_path, detected, requirements, + candidate_order=_openai_candidate_order(catalog, source), region="unknown", ) - mantle_requires_region_change = mantle is not None - if mantle: - model_key, model, path_config = mantle - reason = ( - "Preserves the OpenAI SDK and Chat Completions surface; gives up " - "runtime-only Bedrock governance." - if mantle_path == "mantle_openai_chat" - else "Preserves the OpenAI SDK and Responses surface; gives up " - "runtime-only Bedrock governance." + requires_region_change = openai_hit is not None + if openai_hit: + model_key, model, path_config = openai_hit + api_label = ( + "Chat Completions" + if openai_path in {"mantle_openai_chat", "runtime_openai_chat"} + else "Responses" ) - if mantle_requires_region_change: + if selector: + reason = ( + f"Preserves the OpenAI SDK and the {api_label} surface on bedrock-mantle, " + f"selected by mantle_selector {selector}; gives up bedrock-runtime-only " + "governance (Guardrails, invocation logging)." + ) + else: + reason = ( + "Keeps the OpenAI SDK and the source API on bedrock-runtime via a CRIS id " + "(invocation logging and CloudWatch included); Guardrails and application " + "inference profiles are evidenced only on Converse for this model — see its card." + ) + if requires_region_change: availability = model.get("region_availability") or {} supported_regions = sorted( set(availability.get("commercial", [])) @@ -893,17 +1007,20 @@ def _decision_options(catalog, workload, region): f" Requires changing from {region} to a catalog-supported Mantle region " f"({region_label})." ) + invocation_model_id = _resolve_invocation_model_id( + path_config["model_id"], path_config["requires_cris"], requirements, model, region, + ) options.append( { "model_key": model_key, "model": path_config["model_id"], - "api_path": mantle_path, - "invocation_model_id": _resolve_invocation_model_id( - path_config["model_id"], path_config["requires_cris"], workload["requirements"], - model, region, + "api_path": openai_path, + "invocation_model_id": invocation_model_id, + "requires_cris": anthropic_model_recommendation._invocation_requires_cris( + path_config, invocation_model_id ), - "requires_cris": path_config["requires_cris"], "reason": reason, + "mantle_selector": selector if openai_path.startswith("mantle_") else None, } ) runtime, _ = _catalog_model_for_path( @@ -934,6 +1051,7 @@ def _decision_options(catalog, workload, region): else "Uses Bedrock-native Converse request/response shapes and a " "cross-family model; requires rewriting the OpenAI integration." ), + "mantle_selector": None, } ) return options @@ -955,6 +1073,7 @@ def recommend_openai_workload(workload, region, catalog): surface = source.get("api_surface") continuity = _api_continuity(requirements) runtime_required = _runtime_required(requirements) + converse_required = _converse_required(requirements) continuity_required = continuity == "required" feature_assessment = _feature_assessment(workload) @@ -962,6 +1081,7 @@ def recommend_openai_workload(workload, region, catalog): # the merged assessment marks it detected. Explicit absent/unknown are excluded, # so downstream compatibility/deltas/targets never contradict the assessment. detected = [f for f, status in feature_assessment.items() if status == "detected"] + selector = _mantle_selector(workload, feature_assessment) unknown_required = _unknown_required_features(feature_assessment, requirements) @@ -978,8 +1098,9 @@ def _unresolved( "model_identity": None, "api_path": None, "invocation_model_id": None, + "mantle_selector": None, "decision_options": ( - _decision_options(catalog, workload, region) + _decision_options(catalog, workload, region, selector) if decision_options is None else decision_options ), @@ -1021,35 +1142,65 @@ def _unresolved( "", ) - # --- Hard conflict: OpenAI continuity required AND runtime-only governance --- - if continuity_required and runtime_required: - options = _decision_options(catalog, workload, region) + def _decision_conflict(reason_head, reasons, message, remediation, options): if len(options) < 2: raise ValueError( f"catalog cannot supply both decision options for workload " f"{workload['workload_id']}" ) return _unresolved( - "Required OpenAI API continuity conflicts with a runtime-only Bedrock " - "governance requirement.", + reason_head, reasons, + _finding("model_path_decision_required", "[BLOCKS]", message, remediation), + "", decision_options=options, + ) + + # --- Hard conflict: a verified Mantle-only need AND bedrock-runtime-only governance --- + if selector and runtime_required: + options = _decision_options(catalog, workload, region, selector) + if not converse_required: + # Logging/CloudWatch governance is satisfied on any bedrock-runtime path, so + # the runtime option keeps the OpenAI API rather than switching to Converse. + options = [ + option for option in options if option["api_path"].startswith("mantle_") + ] + [ + option for option in _decision_options(catalog, workload, region, None) + if option["api_path"].startswith("runtime_openai_") + ] + return _decision_conflict( + f"A verified Mantle-only need (mantle_selector {selector}) conflicts with a " + "bedrock-runtime-only governance requirement.", [ - "Mantle preserves the OpenAI SDK and the supported source API surface; runtime " - "Converse provides Bedrock-native governance and keeps an exact same-model " - "runtime candidate when the catalog provides one." + "bedrock-mantle serves the selected capability; bedrock-runtime provides " + "Guardrails, invocation logging and application inference profiles " + "(bedrock-endpoint-policy.md)." ], - _finding( - "model_path_decision_required", - "[BLOCKS]", - "OpenAI API continuity and runtime-only governance cannot both be satisfied " - "by one path.", - "Choose the Mantle continuity option or the runtime governance option, " - "update requirements, and rerun Model Recommend.", - ), - "", + f"A verified Mantle-only need (mantle_selector {selector}) and " + "bedrock-runtime-only governance cannot both be satisfied by one path.", + "Choose the Mantle option or the bedrock-runtime governance option, update " + "requirements, and rerun Model Recommend.", + options, + ) + + # --- Hard conflict: OpenAI continuity required AND Converse-only governance --- + if continuity_required and converse_required: + return _decision_conflict( + "Required OpenAI API continuity conflicts with Converse-only governance " + "(Guardrails / multi-model Converse).", + [ + "bedrock-runtime Responses/Chat Completions keep the OpenAI SDK and the " + "source API via a CRIS id; runtime Converse provides Guardrails and the " + "multi-model surface and keeps an exact same-model runtime candidate when " + "the catalog provides one." + ], + "Required OpenAI API continuity and Converse-only governance (Guardrails / " + "multi-model Converse) cannot both be satisfied by one bedrock-runtime API.", + "Choose the bedrock-runtime OpenAI API option or the runtime Converse " + "governance option, update requirements, and rerun Model Recommend.", + _decision_options(catalog, workload, region, None), ) # --- Select a path --- - if runtime_required: + if converse_required: path = "runtime_converse" candidate_order = _converse_candidate_order(source, requirements) same = _same_model_runtime_key(source) @@ -1068,48 +1219,58 @@ def _unresolved( "(capability-evidence fallback across Claude tiers). Sources without an " "exact runtime catalog identity require this family switch." ) - else: + elif selector: path = _mantle_path(source, requirements) - candidate_order = _mantle_candidate_order(catalog, source) - if path == "mantle_openai_chat": - rationale_head = ( - "GPT-6 Chat Completions API continuity requires an in-region Mantle Chat " - "Completions model." - ) - else: - rationale_head = ( - "OpenAI source lands on Mantle Responses, with the exact source model " - "evaluated before capability and numeric fallbacks." - ) + candidate_order = _openai_candidate_order(catalog, source) + rationale_head = ( + f"Mantle {_api_label(path)} selected by mantle_selector {selector} " + "(bedrock-endpoint-policy.md); the exact source model is evaluated first." + ) + else: + path = _runtime_openai_path(surface) + candidate_order = _openai_candidate_order(catalog, source) + rationale_head = ( + f"OpenAI source stays on bedrock-runtime {_api_label(path)} with the exact " + "source model evaluated before capability and numeric fallbacks " + "(bedrock-endpoint-policy.md default)." + ) # Astra's new paths must not inherit the older GPT-5.x defaults. Keep this # source on Astra and fail closed when the required path lacks evidence. - if _same_model_runtime_key(source) == "openai_gpt_6_astra": + astra = _same_model_runtime_key(source) == "openai_gpt_6_astra" + if astra: candidate_order = ["openai_gpt_6_astra"] - preferred = requirements.get("preferred_api_path") - if preferred: - available = catalog["models"]["openai_gpt_6_astra"]["paths"].get(preferred, {}) - if (available.get("available") is not True - or runtime_required and preferred != "runtime_converse" - or continuity_required and preferred == "runtime_converse"): - return _unresolved( - "The explicit Astra API preference conflicts with the available paths " - "or required API/governance constraints.", [], - _finding( - "preferred_api_path_conflict", "[BLOCKS]", - f"The requested {preferred} path cannot satisfy this workload.", - "Choose a supported API or revise the conflicting requirements, " - "then rerun Model Recommend.", - ), preferred, - ) - path = preferred - elif not runtime_required and surface == "chat_completions": - path = "mantle_openai_chat" rationale_head = ( "Keep GPT-6 Astra on Bedrock using the compatible explicit API preference, " "otherwise the source-compatible API or required runtime Converse path; " "apply Astra's own region and feature evidence." ) + preferred = requirements.get("preferred_api_path") + if preferred: + first = catalog["models"].get(candidate_order[0], {}) if candidate_order else {} + available = first.get("paths", {}).get(preferred, {}) + if (available.get("available") is not True + or runtime_required and not preferred.startswith("runtime_") + or converse_required and preferred != "runtime_converse" + or continuity_required and preferred == "runtime_converse" + or preferred.startswith("mantle_") and not selector): + return _unresolved( + "The explicit API preference conflicts with the available paths, the " + "endpoint policy, or required API/governance constraints.", [], + _finding( + "preferred_api_path_conflict", "[BLOCKS]", + f"The requested {preferred} path cannot satisfy this workload." + + ( + " A bedrock-mantle path needs a mantle_selector " + "(bedrock-endpoint-policy.md)." + if preferred.startswith("mantle_") and not selector + else "" + ), + "Choose a supported API or revise the conflicting requirements, " + "then rerun Model Recommend.", + ), preferred, + ) + path = preferred # Evidence-driven selection: pick the first candidate on the path whose catalog # capabilities cover the required/detected text features (a newer model without @@ -1139,16 +1300,66 @@ def _unresolved( ) raise ValueError(f"catalog has no available model for path {path}") model_key, model, path_config = catalog_hit - if path == "mantle_openai_chat": + + # Derived selector: strict in-Region processing on a CRIS-only runtime candidate + # is a Mantle-only need (bedrock-endpoint-policy.md in_region_processing), unless + # bedrock-runtime-only governance pins the workload to runtime. + if ( + path.startswith("runtime_") + and requirements.get("data_residency") == "in_region_required" + and path_config["requires_cris"] + and not anthropic_model_recommendation._runtime_in_region_allowed( + path_config["model_id"], model, region, requirements + ) + and not runtime_required + ): + selector = "in_region_processing" + path = _mantle_path(source, requirements) + candidate_order = ["openai_gpt_6_astra"] if astra else _openai_candidate_order(catalog, source) + catalog_hit, unsupported = _catalog_model_for_path( + catalog, path, detected, requirements, candidate_order=candidate_order, + region=region, + ) + if not catalog_hit: + return _unresolved( + rationale_head, + [ + f"The catalog has no {path} candidate available in {region}; " + "select another region or migration path." + ], + _finding( + "mantle_region_unavailable", + "[BLOCKS]", + f"No in-region {path} model is available in {region}.", + "Choose a region in the selected model's Mantle footprint, choose " + "runtime Converse, or update the continuity requirement, then rerun " + "Model Recommend.", + ), + path, + ) + model_key, model, path_config = catalog_hit + rationale_head = ( + "Strict in-Region processing was requested and bedrock-runtime serves this " + f"model only through a CRIS profile; Mantle {_api_label(path)} keeps processing " + "in-Region (mantle_selector in_region_processing)." + ) + + if path in {"mantle_openai_chat", "runtime_openai_chat"}: + endpoint = "Mantle" if path.startswith("mantle_") else "bedrock-runtime" if model_key == _same_model_catalog_key(source): rationale_head = ( - "GPT-6 Chat Completions source with API continuity lands on the exact " - "same model through Mantle Chat Completions." + "Chat Completions source lands on the exact same model through " + f"{endpoint} Chat Completions" + + ( + f" (mantle_selector {selector})." + if path.startswith("mantle_") + else " (bedrock-endpoint-policy.md default)." + ) ) else: rationale_head = ( f"The exact source model is unavailable in {region}; " - f"{model['display_name']} is the in-region Mantle Chat Completions fallback." + f"{model['display_name']} is the in-region {endpoint} Chat Completions fallback." ) if unsupported: return _unresolved( @@ -1248,40 +1459,54 @@ def _unresolved( ) (blocks if finding["tag"] == "[BLOCKS]" else tuning).append(finding) - if path == "runtime_converse" and path_config["requires_cris"] and invocation_model_id is None: - # GPT-5.6 on bedrock-runtime is CRIS-only — there is no in-region invocation - # form. A residency posture that forbids both Global and Geo CRIS makes the - # same-model governance path unusable; say so explicitly instead of shipping - # a recommendation with an unresolvable invocation id. - blocks.append( - _finding( - "cris_residency_unresolved", - "[BLOCKS]", - "This runtime path is CRIS-only, and the stated data-residency posture " - "permits neither Global nor Geo cross-region inference, so no invocation " - "id can be resolved.", - "Relax residency to Global or a Geo CRIS geography, supply an explicit " - "inference_profile_id, or take the Claude cross-family Converse path.", + if path.startswith("runtime_") and path_config["requires_cris"] and invocation_model_id is None: + # GPT models on bedrock-runtime are CRIS-only — there is no in-region invocation + # form. Unresolved residency, or a posture that forbids both Global and Geo + # CRIS, leaves no resolvable invocation id; say so explicitly instead of + # shipping a recommendation with an unresolvable invocation id. + if requirements.get("data_residency", "unknown") == "unknown": + blocks.append( + _finding( + "cris_residency_unresolved", + "[BLOCKS]", + "This bedrock-runtime path is CRIS-only and data_residency is unresolved, " + "so no invocation id can be resolved.", + "Record data_residency (global_allowed, or geo_required with " + "cris_geography) or supply an explicit inference_profile_id, then rerun " + "Model Recommend.", + ) + ) + else: + blocks.append( + _finding( + "cris_residency_unresolved", + "[BLOCKS]", + "This runtime path is CRIS-only, and the stated data-residency posture " + "permits neither Global nor Geo cross-region inference, so no invocation " + "id can be resolved.", + "Relax residency to Global or a Geo CRIS geography, supply an explicit " + "inference_profile_id, or take the Claude cross-family Converse path.", + ) ) - ) - if path == "mantle_openai_responses" and surface == "chat_completions": + if path in {"mantle_openai_responses", "runtime_openai_responses"} and surface == "chat_completions": + endpoint = "Mantle" if path.startswith("mantle_") else "bedrock-runtime" deltas.extend(_chat_to_responses_deltas()) blocks.append( _finding( "chat_completions_to_responses_required", "[BLOCKS]", ( - "The selected Astra Responses path requires reshaping the Chat source." + f"The selected Astra {endpoint} Responses path requires reshaping the Chat source." if model_key == "openai_gpt_6_astra" - else "The selected Mantle Responses path requires reshaping the Chat source " + else f"The selected {endpoint} Responses path requires reshaping the Chat source " "to the Responses API (Chat Completions support is per-model: see the " - "selected model's mantle_openai_chat catalog entry)." + "selected model's Chat Completions catalog entry)." ), "Apply the request/response/tool/state reshape deltas before cutover.", ) ) - if path == "mantle_openai_chat" and surface == "responses": + if path in {"mantle_openai_chat", "runtime_openai_chat"} and surface == "responses": deltas.extend([ _delta( "responses_to_chat_request", "path", @@ -1296,7 +1521,11 @@ def _unresolved( ]) blocks.append(_finding( "responses_to_chat_completions_required", "[BLOCKS]", - "The selected Astra Chat path requires reshaping the Responses source.", + ( + "The selected Astra Chat path requires reshaping the Responses source." + if model_key == "openai_gpt_6_astra" + else "The selected Chat Completions path requires reshaping the Responses source." + ), "Apply the request/response/tool/state mapping before cutover.", )) deltas.extend(f_deltas) @@ -1333,6 +1562,7 @@ def _unresolved( "model_identity": _model_identity(model_key, model, path_config, invocation_model_id), "api_path": path, "invocation_model_id": invocation_model_id, + "mantle_selector": selector if path.startswith("mantle_") else None, "decision_options": [], "alternatives": [], "rationale": rationale, diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-recommendation.json b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-recommendation.json index 56f24e7a1..adfe15f74 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-recommendation.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/model-recommendation.json @@ -315,7 +315,8 @@ "migration_deltas", "evaluation", "rollout", - "verification" + "verification", + "mantle_selector" ], "properties": { "workload_id": { @@ -549,6 +550,24 @@ } } } + }, + { + "if": { + "properties": { + "api_path": { + "type": "string", + "pattern": "^mantle_" + } + }, + "required": ["api_path"] + }, + "then": { + "properties": { + "mantle_selector": { + "type": "string" + } + } + } } ], "additionalProperties": false diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py index 33c5b9d9e..b1f4810b8 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py @@ -1,4 +1,11 @@ -"""Tests for the OpenAI-to-Bedrock provider module (handoff Section 11 matrix).""" +"""Tests for the OpenAI-to-Bedrock provider module (handoff Section 11 matrix). + +Endpoint rule under test (skills/shared/ai/bedrock-endpoint-policy.md): the OpenAI +source stays on bedrock-runtime Responses / Chat Completions on the same model by +default; a bedrock-mantle path is selected only by a mantle_selector (explicit, or +derived from a detected Web Search tool / strict in-Region processing); Guardrails +and multi-model Converse select runtime Converse. +""" import json import pathlib @@ -53,36 +60,97 @@ def _delta_codes(rec): # --- 11.1 Provider and path selection ------------------------------------- +# +# bedrock-runtime is the default endpoint (bedrock-endpoint-policy.md); a +# bedrock-mantle path is selected only by a mantle_selector. API continuity alone +# never selects Mantle. -def test_responses_source_with_continuity_recommends_mantle_responses(): +def test_default_responses_source_lands_on_runtime_responses_same_model(): + rec = _recommend() + assert rec["provider_module"] == "openai" + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "runtime_openai_responses" + assert rec["primary_model"] == "openai.gpt-5.4" + assert rec["mantle_selector"] is None + assert rec["model_identity"]["requires_cris"] is True + # Residency is unknown, so no CRIS prefix is guessed; the block says why. + assert rec["invocation_model_id"] is None + block = next(b for b in rec["blocks"] if b["code"] == "cris_residency_unresolved") + assert "unresolved" in block["message"] + assert "data_residency" in block["remediation"] + + +def test_default_with_global_residency_resolves_global_cris_id(): + rec = _recommend(_workload(requirements={"data_residency": "global_allowed"})) + assert rec["api_path"] == "runtime_openai_responses" + assert rec["invocation_model_id"] == "global.openai.gpt-5.4" + assert "cris_residency_unresolved" not in _codes(rec["blocks"]) + + +def test_responses_source_with_continuity_lands_on_runtime_responses(): + # Negative test: Responses-only continuity selects bedrock-runtime, not Mantle. rec = _recommend( _workload(requirements={"api_continuity": "required"}) ) - assert rec["provider_module"] == "openai" assert rec["decision_status"] == "recommended" - assert rec["api_path"] == "mantle_openai_responses" - assert rec["primary_model"] == "openai.gpt-5.6-sol" + assert rec["api_path"] == "runtime_openai_responses" + assert rec["primary_model"] == "openai.gpt-5.4" + assert rec["mantle_selector"] is None + assert "chat_completions_to_responses_required" not in _codes(rec["blocks"]) -def test_chat_completions_unrecognized_gpt5_source_lands_on_sol_responses(): - # The default workload's source is `gpt-5.4`, which the engine does not yet - # recognize as a same-model identity (see _SOURCE_MODEL_KEYS), so it lands on - # the first catalog candidate — GPT-5.6 Sol — whose card leaves Chat - # Completions unchecked on Mantle. The reshape here follows from THAT model's - # card, not from a blanket "GPT-5.x on Mantle is Responses-only" rule: the - # GPT-5.4 card itself lists Chat Completions on Mantle (see the catalog test - # below). +def test_chat_completions_continuity_lands_on_runtime_chat_same_model(): + # gpt-5.4 is a recognized same-model identity; its card lists Chat Completions + # on bedrock-runtime, so the Chat source stays on itself with no reshape. rec = _recommend( _workload( source={"api_surface": "chat_completions"}, requirements={"api_continuity": "required"}, ) ) - assert rec["primary_model"] == "openai.gpt-5.6-sol" - assert OPENAI_CATALOG["models"]["openai_gpt_5_6_sol"]["paths"]["mantle_openai_chat"]["available"] is False + assert rec["decision_status"] == "recommended" + assert rec["primary_model"] == "openai.gpt-5.4" + assert rec["api_path"] == "runtime_openai_chat" + assert rec["mantle_selector"] is None + assert "chat_completions_to_responses_required" not in _codes(rec["blocks"]) + assert "responses_no_n" not in _delta_codes(rec) + + +def test_continuity_plus_logging_governance_stays_on_runtime_openai_api(): + # invocation_logging / cloudwatch are satisfied on any bedrock-runtime path. + rec = _recommend( + _workload( + requirements={ + "api_continuity": "required", + "governance": ["invocation_logging", "cloudwatch"], + } + ) + ) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "runtime_openai_responses" + assert rec["primary_model"] == "openai.gpt-5.4" + + +@pytest.mark.parametrize("selector", oai.MANTLE_SELECTORS) +def test_each_mantle_selector_selects_mantle_and_is_echoed(selector): + rec = _recommend(_workload(requirements={"mantle_selector": selector})) + assert rec["decision_status"] == "recommended" assert rec["api_path"] == "mantle_openai_responses" - assert rec["api_path"] != "mantle_openai_chat" - assert "chat_completions_to_responses_required" in _codes(rec["blocks"]) + assert rec["primary_model"] == "openai.gpt-5.4" + assert rec["invocation_model_id"] == "openai.gpt-5.4" + assert rec["mantle_selector"] == selector + + +def test_mantle_selectors_match_the_input_schema_enum(): + schema = json.loads((SCRIPTS / "schemas" / "model-recommendation-input.json").read_text()) + enum = schema["properties"]["workloads"]["items"]["properties"]["requirements"][ + "properties"]["mantle_selector"]["enum"] + assert list(oai.MANTLE_SELECTORS) == enum + + +def test_unknown_mantle_selector_is_rejected(): + with pytest.raises(ValueError, match="mantle_selector"): + _recommend(_workload(requirements={"mantle_selector": "because"})) @pytest.mark.parametrize( @@ -136,14 +204,26 @@ def test_runtime_converse_path_text_does_not_grant_in_profile_to_gpt54_gpt55(): assert "Guardrails and application inference profiles are Converse-only" in constraints -def test_chat_completions_case_includes_full_reshape_deltas(): +@pytest.mark.parametrize("source_id,requirements,path", [ + # A Chat source explicitly moved to bedrock-runtime Responses. + ("gpt-5.4", {"preferred_api_path": "runtime_openai_responses"}, "runtime_openai_responses"), + # A Chat source on Mantle Responses by selector + preference (GPT-5.6 Sol's + # Mantle Chat row is unavailable, so Responses is its only Mantle API). + ("gpt-5.6-sol", {"preferred_api_path": "mantle_openai_responses", + "mantle_selector": "background_inference"}, "mantle_openai_responses"), +]) +def test_chat_completions_case_includes_full_reshape_deltas(source_id, requirements, path): rec = _recommend( _workload( - source={"api_surface": "chat_completions"}, - requirements={"api_continuity": "required"}, + source={"model_ids": [source_id], "api_surface": "chat_completions"}, + requirements=requirements, detected_features=["tool_or_function_calling", "conversation_state"], - ) + ), + region="us-east-1", ) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == path + assert "chat_completions_to_responses_required" in _codes(rec["blocks"]) codes = _delta_codes(rec) assert "chat_to_responses_request" in codes assert "chat_to_responses_response" in codes @@ -151,9 +231,29 @@ def test_chat_completions_case_includes_full_reshape_deltas(): assert "chat_to_responses_state" in codes +def test_chat_source_with_selector_keeps_chat_on_an_available_mantle_chat_model(): + # GPT-5.6 Sol has no Mantle Chat row; the walk keeps the Chat surface on the + # next same-family model whose Mantle Chat row is in region, not a reshape. + rec = _recommend( + _workload( + source={"model_ids": ["gpt-5.6-sol"], "api_surface": "chat_completions"}, + requirements={"mantle_selector": "background_inference"}, + ), + region="us-east-1", + ) + assert rec["api_path"] == "mantle_openai_chat" + assert rec["primary_model"] == "openai.gpt-5.5" + assert "fallback" in rec["rationale"][0] + assert "chat_completions_to_responses_required" not in _codes(rec["blocks"]) + + def test_governance_selects_runtime_converse(): + # Cross-family Converse tier: an unrecognized source id (gpt-4) with Guardrails. rec = _recommend( - _workload(requirements={"governance": ["guardrails", "invocation_logging"]}) + _workload( + source={"model_ids": ["gpt-4"]}, + requirements={"governance": ["guardrails", "invocation_logging"]}, + ) ) assert rec["decision_status"] == "recommended" assert rec["api_path"] == "runtime_converse" @@ -162,9 +262,10 @@ def test_governance_selects_runtime_converse(): "Claude Sonnet 5 (Converse tier match for GPT-5.6 Terra / 5.5 / 5.4)" ) assert rec["model_identity"]["version"] == "5" + assert rec["mantle_selector"] is None -def test_continuity_plus_runtime_only_returns_decision_required(): +def test_continuity_plus_converse_only_governance_returns_two_runtime_options(): rec = _recommend( _workload( requirements={ @@ -176,11 +277,14 @@ def test_continuity_plus_runtime_only_returns_decision_required(): assert rec["decision_status"] == "decision_required" assert rec["primary_model"] is None assert rec["api_path"] is None + assert rec["mantle_selector"] is None assert {o["api_path"] for o in rec["decision_options"]} == { - "mantle_openai_responses", + "runtime_openai_responses", "runtime_converse", } + assert all(o["mantle_selector"] is None for o in rec["decision_options"]) assert "model_path_decision_required" in _codes(rec["blocks"]) + assert "Converse-only governance" in rec["blocks"][0]["message"] def test_decision_options_explain_tradeoffs(): @@ -190,10 +294,102 @@ def test_decision_options_explain_tradeoffs(): ) ) reasons = {o["api_path"]: o["reason"] for o in rec["decision_options"]} - assert "OpenAI SDK" in reasons["mantle_openai_responses"] + assert "OpenAI SDK" in reasons["runtime_openai_responses"] + assert "Converse" in reasons["runtime_openai_responses"] + # gpt-5.4 is a recognized same-model identity: Converse keeps the model. + assert "SAME-MODEL" in reasons["runtime_converse"] + + +def test_cross_family_converse_option_names_the_bedrock_native_rewrite(): + rec = _recommend( + _workload( + source={"model_ids": ["gpt-4"]}, + requirements={"api_continuity": "required", "governance": ["guardrails"]}, + ) + ) + reasons = {o["api_path"]: o["reason"] for o in rec["decision_options"]} assert "Bedrock-native" in reasons["runtime_converse"] +# --- Web Search: a Mantle-only server-side tool (mantle_selector web_search) -- + +def test_detected_web_search_derives_the_mantle_selector(): + rec = _recommend(_workload(detected_features=["web_search"])) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "mantle_openai_responses" + assert rec["mantle_selector"] == "web_search" + impacts = {a["feature"]: a["impact"] for a in rec["architecture_impacts"]} + assert "bedrock-mantle" in impacts["web_search"] + assert "passthrough" not in impacts["web_search"].lower() + assert "docs.aws.amazon.com" in impacts["web_search"] + assert "web_search" not in rec["compatibility"]["rearchitecture"] + assert "web_search" in rec["compatibility"]["portable"] + + +def test_web_search_plus_guardrails_offers_a_mantle_option_with_the_selector(): + rec = _recommend( + _workload(detected_features=["web_search"], requirements={"governance": ["guardrails"]}) + ) + assert rec["decision_status"] == "decision_required" + assert "model_path_decision_required" in _codes(rec["blocks"]) + assert "web_search" in rec["blocks"][0]["message"] + mantle = next(o for o in rec["decision_options"] if o["api_path"].startswith("mantle_")) + assert mantle["api_path"] == "mantle_openai_responses" + assert mantle["mantle_selector"] == "web_search" + assert {o["api_path"] for o in rec["decision_options"]} == { + "mantle_openai_responses", "runtime_converse", + } + + +def test_web_search_plus_logging_governance_offers_the_runtime_openai_api(): + rec = _recommend( + _workload(detected_features=["web_search"], + requirements={"governance": ["invocation_logging"]}) + ) + assert rec["decision_status"] == "decision_required" + assert {o["api_path"] for o in rec["decision_options"]} == { + "mantle_openai_responses", "runtime_openai_responses", + } + + +def test_web_search_marked_absent_does_not_select_mantle(): + rec = _recommend( + _workload(detected_features=["web_search"], feature_status={"web_search": "absent"}) + ) + assert rec["api_path"] == "runtime_openai_responses" + assert rec["mantle_selector"] is None + assert "web_search" not in rec["compatibility"]["portable"] + + +def test_web_search_on_a_runtime_path_stays_rearchitecture(): + compat = oai._compatibility(["web_search"], {}, "runtime_openai_responses", + OPENAI_CATALOG["models"]["openai_gpt_5_4"]) + assert "web_search" in compat["rearchitecture"] + assert "web_search" not in compat["portable"] + + +# --- Derived in_region_processing ------------------------------------------ + +def test_in_region_required_on_cris_only_runtime_model_selects_mantle(): + rec = _recommend(_workload(requirements={"data_residency": "in_region_required"})) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "mantle_openai_responses" + assert rec["primary_model"] == "openai.gpt-5.4" + assert rec["invocation_model_id"] == "openai.gpt-5.4" + assert rec["mantle_selector"] == "in_region_processing" + assert "in_region_processing" in rec["rationale"][0] + assert "cris_residency_unresolved" not in _codes(rec["blocks"]) + + +def test_in_region_required_with_guardrails_stays_on_runtime_and_blocks(): + rec = _recommend(_workload(requirements={ + "data_residency": "in_region_required", "governance": ["guardrails"], + })) + assert rec["api_path"] == "runtime_converse" + assert rec["mantle_selector"] is None + assert "cris_residency_unresolved" in _codes(rec["blocks"]) + + # --- GPT-6 exact-model routing -------------------------------------------- _GPT6_MODELS = ( @@ -205,21 +401,26 @@ def test_decision_options_explain_tradeoffs(): @pytest.mark.parametrize(("source_id", "target_id"), _GPT6_MODELS) +@pytest.mark.parametrize(("selector", "path"), [ + (None, "runtime_openai_responses"), + ("server_side_tools", "mantle_openai_responses"), +]) def test_gpt6_supported_region_selects_exact_model_and_validates_schema( - source_id, target_id + source_id, target_id, selector, path ): critical = ["image_input_vision"] if source_id in {"gpt-6-sol", "gpt-6-luna"} else [] - workload = _workload( - source={"model_ids": [source_id]}, - requirements={"api_continuity": "required", "critical_features": critical}, - ) + requirements = {"api_continuity": "required", "critical_features": critical} + if selector: + requirements["mantle_selector"] = selector + workload = _workload(source={"model_ids": [source_id]}, requirements=requirements) out = _full_recommend(workload, region="us-east-1") jsonschema.validate(out, _OUT_SCHEMA) rec = out["workloads"]["chat-svc"] assert oai.detect_family(source_id) == "reasoning" assert rec["decision_status"] == "recommended" assert rec["primary_model"] == target_id - assert rec["api_path"] == "mantle_openai_responses" + assert rec["api_path"] == path + assert rec["mantle_selector"] == selector if critical: assert "image_input_vision" in rec["compatibility"]["native"] @@ -301,51 +502,49 @@ def test_active_catalog_contains_gpt6_profile_metadata_and_current_provenance(): @pytest.mark.parametrize(("source_id", "target_id"), _GPT6_MODELS) -def test_gpt6_chat_completions_preserves_chat_surface(source_id, target_id): +@pytest.mark.parametrize(("selector", "path"), [ + (None, "runtime_openai_chat"), + ("server_side_tools", "mantle_openai_chat"), +]) +def test_gpt6_chat_completions_preserves_chat_surface(source_id, target_id, selector, path): + requirements = {"api_continuity": "required"} + if selector: + requirements["mantle_selector"] = selector rec = _recommend( _workload( source={"model_ids": [source_id], "api_surface": "chat_completions"}, - requirements={"api_continuity": "required"}, + requirements=requirements, ), region="us-east-1", ) assert rec["primary_model"] == target_id - assert rec["api_path"] == "mantle_openai_chat" + assert rec["api_path"] == path assert "chat_completions_to_responses_required" not in _codes(rec["blocks"]) def test_gpt6_mantle_skips_exact_model_outside_cataloged_region(): + # With a selector, the Mantle walk skips GPT-6 Sol outside its footprint and + # lands on the same-family model in region (Astra), never on GPT-5.x. rec = _recommend( _workload( source={"model_ids": ["gpt-6-sol"]}, - requirements={"api_continuity": "required"}, + requirements={"api_continuity": "required", "mantle_selector": "server_side_tools"}, ), region="us-west-2", ) assert rec["decision_status"] == "recommended" - assert rec["primary_model"] != "openai.gpt-6-sol" - assert "fallback" in rec["rationale"][0] + assert rec["api_path"] == "mantle_openai_responses" + assert rec["primary_model"] == "openai.gpt-6-astra" assert "exact same model" not in rec["rationale"][0] -# Deliberate consequence of recording the public GPT-5.5/GPT-5.4 facts (Chat -# Completions available on Mantle, cataloged region_availability): the Mantle Chat -# fallback walks candidates in catalog order, and GPT-5.5 / GPT-5.4 sit ahead of -# GPT-6 Astra in that order. A GPT-6 Sol / 6.1 Sol Chat source outside its own -# footprint therefore lands on GPT-5.5 (us-east-2) or GPT-5.4 (us-west-2) instead -# of Astra / a blocking decision. Pinned here so the shift is visible; reordering -# the candidate walk is engine work tracked for the follow-up PR. +# Replaces the PR 1 "deliberate consequence" tests: the candidate walk is now +# same model -> same family (newest first) -> older families (newest first), and +# bedrock-runtime is the default, so a GPT-6 Chat source outside its Mantle +# footprint stays on itself unless a selector puts it on Mantle. @pytest.mark.parametrize("source_id", ["gpt-6-sol", "gpt-6.1-sol"]) -@pytest.mark.parametrize( - ("region", "fallback_id", "fallback_key"), - [ - ("us-east-2", "openai.gpt-5.5", "openai_gpt_5_5"), - ("us-west-2", "openai.gpt-5.4", "openai_gpt_5_4"), - ], -) -def test_gpt6_chat_outside_own_footprint_falls_back_to_in_region_gpt5_chat( - source_id, region, fallback_id, fallback_key -): +@pytest.mark.parametrize("region", ["us-east-2", "us-west-2"]) +def test_gpt6_chat_without_selector_stays_on_itself_via_runtime_chat(source_id, region): rec = _recommend( _workload( source={"model_ids": [source_id], "api_surface": "chat_completions"}, @@ -354,35 +553,93 @@ def test_gpt6_chat_outside_own_footprint_falls_back_to_in_region_gpt5_chat( region=region, ) assert rec["decision_status"] == "recommended" - assert rec["primary_model"] == fallback_id - assert rec["api_path"] == "mantle_openai_chat" - assert "mantle_region_unavailable" not in _codes(rec["blocks"]) + assert rec["api_path"] == "runtime_openai_chat" + assert rec["primary_model"] == f"openai.{source_id}" + assert rec["mantle_selector"] is None assert "chat_completions_to_responses_required" not in _codes(rec["blocks"]) - assert "fallback" in rec["rationale"][0] - assert region in OPENAI_CATALOG["models"][fallback_key]["region_availability"]["commercial"] -def test_gpt56_sol_responses_outside_us_east_skips_gpt55_for_in_region_terra(): - # GPT-5.5's cataloged footprint is us-east-1/us-east-2 only, so in us-west-2 - # the Responses fallback now skips it and lands on GPT-5.6 Terra (base - # catalog, with no GPT-5.5 region data, selected GPT-5.5 here). +@pytest.mark.parametrize("source_id", ["gpt-6-sol", "gpt-6.1-sol"]) +def test_gpt6_chat_with_selector_in_us_west_2_falls_back_to_same_family_astra(source_id): rec = _recommend( _workload( - source={"model_ids": ["gpt-5.6-sol"]}, - requirements={"api_continuity": "required"}, + source={"model_ids": [source_id], "api_surface": "chat_completions"}, + requirements={"api_continuity": "required", "mantle_selector": "server_side_tools"}, ), region="us-west-2", ) assert rec["decision_status"] == "recommended" - assert rec["primary_model"] == "openai.gpt-5.6-terra" - assert rec["api_path"] == "mantle_openai_responses" + assert rec["api_path"] == "mantle_openai_chat" + assert rec["primary_model"] == "openai.gpt-6-astra" + assert rec["primary_model"] != "openai.gpt-5.4" + assert rec["mantle_selector"] == "server_side_tools" + assert "fallback" in rec["rationale"][0] + + +@pytest.mark.parametrize("source_id", ["gpt-6-sol", "gpt-6.1-sol"]) +def test_gpt6_chat_with_selector_in_us_east_2_prefers_newest_gpt5_over_gpt54(source_id): + # No GPT-6 model lists Mantle Chat in us-east-2 (Astra: us-east-1/us-west-2), + # so the walk falls to the GPT-5 family, newest first: 5.5, never 5.4. + rec = _recommend( + _workload( + source={"model_ids": [source_id], "api_surface": "chat_completions"}, + requirements={"api_continuity": "required", "mantle_selector": "server_side_tools"}, + ), + region="us-east-2", + ) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "mantle_openai_chat" + assert rec["primary_model"] != "openai.gpt-5.4" + assert rec["primary_model"] == "openai.gpt-5.5" + assert "us-east-2" not in OPENAI_CATALOG["models"]["openai_gpt_6_astra"]["region_availability"]["commercial"] + assert "fallback" in rec["rationale"][0] + + +def test_openai_candidate_order_prefers_same_family_then_newer(): + order = oai._openai_candidate_order(OPENAI_CATALOG, {"model_ids": ["gpt-6-sol"]}) + assert order[0] == "openai_gpt_6_sol" + families = [OPENAI_CATALOG["models"][k]["family"] for k in order] + assert families == sorted(families, key=lambda f: 0 if f == "openai_gpt_6" else 1) + assert set(order) == { + k for k, m in OPENAI_CATALOG["models"].items() if m.get("generation") != "bedrock_native" + } + for family in ("openai_gpt_6", "openai_gpt_5"): + versions = [ + oai._version_tuple(OPENAI_CATALOG["models"][k]["version"]) + for k in order[1:] if OPENAI_CATALOG["models"][k]["family"] == family + ] + assert versions == sorted(versions, reverse=True) + # Sources without a parseable GPT major keep catalog order. + opaque = oai._openai_candidate_order(OPENAI_CATALOG, {"model_ids": ["o3-mini"]}) + assert opaque == [ + k for k, m in OPENAI_CATALOG["models"].items() if m.get("generation") != "bedrock_native" + ] + + +def test_gpt56_sol_responses_in_us_west_2_stays_on_itself_or_falls_to_newest_terra(): + runtime = _recommend( + _workload(source={"model_ids": ["gpt-5.6-sol"]}, requirements={"api_continuity": "required"}), + region="us-west-2", + ) + assert runtime["api_path"] == "runtime_openai_responses" + assert runtime["primary_model"] == "openai.gpt-5.6-sol" + # GPT-5.5's cataloged footprint is us-east-1/us-east-2 only, so a Mantle + # selector in us-west-2 lands on GPT-5.6 Terra (same family, newest in region). + mantle = _recommend( + _workload(source={"model_ids": ["gpt-5.6-sol"]}, + requirements={"mantle_selector": "background_inference"}), + region="us-west-2", + ) + assert mantle["decision_status"] == "recommended" + assert mantle["api_path"] == "mantle_openai_responses" + assert mantle["primary_model"] == "openai.gpt-5.6-terra" assert "us-west-2" not in OPENAI_CATALOG["models"]["openai_gpt_5_5"]["region_availability"]["commercial"] def test_gpt6_chat_outside_every_mantle_footprint_returns_blocking_decision(): workload = _workload( source={"model_ids": ["gpt-6-sol"], "api_surface": "chat_completions"}, - requirements={"api_continuity": "required"}, + requirements={"api_continuity": "required", "mantle_selector": "server_side_tools"}, ) out = _full_recommend(workload, region="eu-west-1") jsonschema.validate(out, _OUT_SCHEMA) @@ -395,6 +652,19 @@ def test_gpt6_chat_outside_every_mantle_footprint_returns_blocking_decision(): "mantle_openai_chat", "runtime_converse", } + # Without a selector the same source stays on bedrock-runtime Chat. + out = _full_recommend( + _workload( + source={"model_ids": ["gpt-6-sol"], "api_surface": "chat_completions"}, + requirements={"api_continuity": "required"}, + ), + region="eu-west-1", + ) + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "runtime_openai_chat" + assert "mantle_region_unavailable" not in _codes(rec["blocks"]) def test_gpt6_chat_conflict_option_names_chat_completions_surface(): @@ -403,6 +673,7 @@ def test_gpt6_chat_conflict_option_names_chat_completions_surface(): source={"model_ids": ["gpt-6-sol"], "api_surface": "chat_completions"}, requirements={ "api_continuity": "required", + "mantle_selector": "server_side_tools", "governance": ["guardrails"], }, ), @@ -414,23 +685,28 @@ def test_gpt6_chat_conflict_option_names_chat_completions_surface(): if option["api_path"] == "mantle_openai_chat" ) assert "Chat Completions surface" in option["reason"] + assert "server_side_tools" in option["reason"] assert "Responses surface" not in option["reason"] + assert option["mantle_selector"] == "server_side_tools" -def test_gpt6_chat_findings_preserve_chat_request_shapes(): +@pytest.mark.parametrize(("selector", "path"), [ + (None, "runtime_openai_chat"), + ("server_side_tools", "mantle_openai_chat"), +]) +def test_gpt6_chat_findings_preserve_chat_request_shapes(selector, path): + requirements = {"api_continuity": "required", "uses_n": True, "uses_hosted_state": True} + if selector: + requirements["mantle_selector"] = selector rec = _recommend( _workload( source={"model_ids": ["gpt-6-sol"], "api_surface": "chat_completions"}, - requirements={ - "api_continuity": "required", - "uses_n": True, - "uses_hosted_state": True, - }, + requirements=requirements, detected_features=["tool_or_function_calling", "conversation_state"], ), region="us-east-1", ) - assert rec["api_path"] == "mantle_openai_chat" + assert rec["api_path"] == path assert "responses_no_n" not in _delta_codes(rec) tool_delta = _delta_by_code(rec, "tool_result_shape") assert "role: tool" in tool_delta["description"] @@ -441,27 +717,35 @@ def test_gpt6_chat_findings_preserve_chat_request_shapes(): assert "previous_response_id" not in state_delta["description"] -def test_gpt6_chat_conflict_outside_mantle_region_is_schema_valid(): +@pytest.mark.parametrize(("selector", "openai_path"), [ + (None, "runtime_openai_chat"), + ("server_side_tools", "mantle_openai_chat"), +]) +def test_gpt6_chat_conflict_outside_mantle_region_is_schema_valid(selector, openai_path): + requirements = {"api_continuity": "required", "governance": ["guardrails"]} + if selector: + requirements["mantle_selector"] = selector workload = _workload( source={"model_ids": ["gpt-6-sol"], "api_surface": "chat_completions"}, - requirements={ - "api_continuity": "required", - "governance": ["guardrails"], - }, + requirements=requirements, ) out = _full_recommend(workload, region="eu-west-1") jsonschema.validate(out, _OUT_SCHEMA) rec = out["workloads"]["chat-svc"] assert rec["decision_status"] == "decision_required" assert {option["api_path"] for option in rec["decision_options"]} == { - "mantle_openai_chat", + openai_path, "runtime_converse", } - mantle = next( + option = next( option for option in rec["decision_options"] - if option["api_path"] == "mantle_openai_chat" + if option["api_path"] == openai_path ) - assert "Requires changing from eu-west-1" in mantle["reason"] + if selector: + assert "Requires changing from eu-west-1" in option["reason"] + assert option["mantle_selector"] == selector + else: + assert option["mantle_selector"] is None def test_gpt6_chat_unknown_required_feature_outside_region_has_two_options(): @@ -506,15 +790,13 @@ def test_runtime_unresolved_branches_outside_mantle_region_are_schema_valid( jsonschema.validate(out, _OUT_SCHEMA) rec = out["workloads"]["chat-svc"] assert rec["decision_status"] == "decision_required" + # Without a selector the OpenAI-API option is bedrock-runtime Chat (no Mantle + # option, no region change); Converse governance is the other side. assert {option["api_path"] for option in rec["decision_options"]} == { - "mantle_openai_chat", + "runtime_openai_chat", "runtime_converse", } - mantle = next( - option for option in rec["decision_options"] - if option["api_path"] == "mantle_openai_chat" - ) - assert "Requires changing from eu-west-1" in mantle["reason"] + assert all(option["mantle_selector"] is None for option in rec["decision_options"]) # --- 11.2 Model-generation analysis --------------------------------------- @@ -560,10 +842,13 @@ def test_reasoning_headroom_is_a_starting_heuristic_not_a_guarantee(): # --- 11.3 Parameter and API behavior -------------------------------------- def test_sampling_is_target_derived_not_source_derived(): - # G01: the Mantle target is openai.gpt-5.4 (accepts sampling), so BOTH an - # early-GPT-5 source and a GPT-5.4 source get the same target-derived finding. + # G01: both targets are GPT-5.2+ on a Responses path (gpt-5.1 -> newest GPT-5 + # in catalog, gpt-5.4 -> itself), so BOTH sources get the same target-derived + # finding. early = _recommend(_workload(source={"model_ids": ["gpt-5.1"]})) later = _recommend(_workload(source={"model_ids": ["gpt-5.4"]})) + assert early["primary_model"] == "openai.gpt-5.6-sol" + assert later["primary_model"] == "openai.gpt-5.4" assert "sampling_params_accepted" in _codes(early["tuning"]) assert "sampling_params_accepted" in _codes(later["tuning"]) # G10: penalties/logprobs/stop are called out as rejected separately. @@ -602,11 +887,13 @@ def _impact_features(rec): return {a["feature"] for a in rec["architecture_impacts"]} -def test_hosted_web_search_not_reported_as_live_native_search(): +def test_hosted_web_search_is_reported_as_a_mantle_server_side_tool(): rec = _recommend(_workload(detected_features=["web_search"])) - impacts = {a["feature"]: a["impact"] for a in rec["architecture_impacts"]} + impacts = {a["feature"]: a for a in rec["architecture_impacts"]} assert "web_search" in impacts - assert "passthrough" in impacts["web_search"].lower() + assert "bedrock-mantle" in impacts["web_search"]["impact"] + assert "mantle_selector web_search" in impacts["web_search"]["recommendation"] + assert "AgentCore Gateway" in impacts["web_search"]["recommendation"] def test_file_search_and_vector_stores_produce_retrieval_impacts(): @@ -648,7 +935,12 @@ def test_unknown_limits_do_not_pass_hard_numeric_requirement(): # Fail-closed invariant: an unknown catalog limit cannot satisfy a hard numeric # need. output_token_ceiling is "unknown" on every entry as of 2026-08-21, so it # probes the invariant the way min_context_tokens no longer can. - rec = _recommend(_workload(requirements={"expected_output_tokens": 100000})) + # GPT-5.6 Sol's output ceiling is still "unknown" (gpt-5.4/5.5 now carry 128000 + # from their cards, so the same-model default would legitimately pass). + model = OPENAI_CATALOG["models"]["openai_gpt_5_6_sol"] + assert model["output_token_ceiling"] == "unknown" + rec = _recommend(_workload(source={"model_ids": ["gpt-5.6-sol"]}, + requirements={"expected_output_tokens": 100000})) assert rec["decision_status"] == "decision_required" assert "unverified_capacity" in _codes(rec["blocks"]) @@ -754,8 +1046,10 @@ def _delta_by_code(rec, code): def test_g01_runtime_target_is_not_reported_as_gpt5(): - # A Bedrock-native (Nova) target must not carry target_version gpt-5.x. - rec = _recommend(_workload(requirements={"governance": ["guardrails"]})) + # A Bedrock-native (Claude) target must not carry target_version gpt-5.x. + rec = _recommend( + _workload(source={"model_ids": ["gpt-4"]}, requirements={"governance": ["guardrails"]}) + ) assert rec["api_path"] == "runtime_converse" assert rec["source_analysis"]["target_version"] == "5" # Claude Sonnet 5, not gpt-5.x assert rec["source_analysis"]["target_version"] != "gpt-5.x" @@ -778,11 +1072,13 @@ def test_g01_legacy_source_to_reasoning_target_marks_generation_change(): def test_g02_runtime_selection_fails_closed_on_unproven_capability(): - # No runtime_converse candidate has streaming evidence in the catalog, so a - # workload requiring it must NOT get a silent native claim — it fails closed. + # Neither GPT-5.6 Terra nor any Claude tier has streaming evidence on + # runtime_converse, so a workload requiring it must NOT get a silent native + # claim — it fails closed. rec = _recommend( _workload( - requirements={"governance": ["guardrails"], "critical_features": ["streaming"]} + source={"model_ids": ["gpt-5.6-terra"]}, + requirements={"governance": ["guardrails"], "critical_features": ["streaming"]}, ) ) assert rec["decision_status"] == "decision_required" @@ -790,48 +1086,88 @@ def test_g02_runtime_selection_fails_closed_on_unproven_capability(): def test_g02_native_only_lists_catalog_evidenced_features(): - # Detected structured output is only feature-probed on gpt-5.4, so the engine - # falls back past gpt-5.6-sol (no structured-output evidence) to gpt-5.4 and - # native lists exactly what that catalog entry supports. + # The same model (gpt-5.4) carries the structured-output evidence, so native + # lists exactly what that catalog entry supports. rec = _recommend( _workload(detected_features=["tool_or_function_calling", "structured_output_json"]) ) assert rec["decision_status"] == "recommended" - assert rec["primary_model"] == "openai.gpt-5.4" # evidence-driven fallback + assert rec["primary_model"] == "openai.gpt-5.4" assert set(rec["compatibility"]["native"]) == { "tool_or_function_calling", "structured_output_json", } +def test_g02_evidence_driven_fallback_skips_unevidenced_newer_models(): + # A gpt-5.1 source has no same-model identity; the walk starts at GPT-5.6 Sol + # (no structured-output evidence) and falls to GPT-5.4, which has it. + rec = _recommend( + _workload(source={"model_ids": ["gpt-5.1"]}, + detected_features=["structured_output_json"]) + ) + assert rec["decision_status"] == "recommended" + assert rec["primary_model"] == "openai.gpt-5.4" + assert "structured_output_json" in rec["compatibility"]["native"] + + def test_g02_vision_requirement_selects_the_evidenced_candidate(): - # image_input_vision is evidenced on gpt-5.6-sol (model card) but not on - # gpt-5.5/terra/luna/5.4 — the engine must select the evidenced candidate, - # never claim native on an unevidenced one. + # image_input_vision is evidenced on the gpt-5.4 card, so the same model is + # selected with a native claim; terra/luna (no vision evidence) are never + # claimed native. rec = _recommend( _workload( requirements={"critical_features": ["image_input_vision"]}, ) ) assert rec["decision_status"] == "recommended" - assert rec["primary_model"] == "openai.gpt-5.6-sol" + assert rec["primary_model"] == "openai.gpt-5.4" assert "image_input_vision" in rec["compatibility"]["native"] + terra = _recommend( + _workload(source={"model_ids": ["gpt-5.6-terra"]}, + requirements={"critical_features": ["image_input_vision"]}) + ) + assert terra["primary_model"] != "openai.gpt-5.6-terra" + assert "image_input_vision" in terra["compatibility"]["native"] -def test_g03_mantle_keeps_typed_responses_parse(): +def test_g03_runtime_responses_keeps_typed_responses_parse(): rec = _recommend(_workload(detected_features=["structured_output_json"])) + assert rec["api_path"] == "runtime_openai_responses" delta = _delta_by_code(rec, "structured_output_text_format") assert delta is not None assert "responses.parse" in delta["description"] + assert "bedrock-runtime" in delta["description"] assert "no direct" not in delta["description"].lower() -def test_g04_mantle_state_offers_hosted_and_manual_modes(): +def test_g03_mantle_keeps_typed_responses_parse(): + rec = _recommend(_workload(detected_features=["structured_output_json"], + requirements={"mantle_selector": "background_inference"})) + assert rec["api_path"] == "mantle_openai_responses" + delta = _delta_by_code(rec, "structured_output_text_format") + assert "responses.parse" in delta["description"] + assert "Mantle" in delta["description"] + + +def test_g04_runtime_state_requires_verification_and_defaults_to_app_owned(): rec = _recommend(_workload(requirements={"uses_hosted_state": True})) + assert rec["api_path"] == "runtime_openai_responses" delta = _delta_by_code(rec, "conversation_state_ownership") assert delta is not None assert "store=True" in delta["description"] assert "store=False" in delta["description"] + assert "bedrock-runtime" in delta["description"] + + +def test_g04_mantle_state_offers_hosted_and_manual_modes(): + rec = _recommend(_workload(requirements={"uses_hosted_state": True, + "mantle_selector": "projects"})) + assert rec["api_path"] == "mantle_openai_responses" + delta = _delta_by_code(rec, "conversation_state_ownership") + assert "store=True" in delta["description"] + assert "store=False" in delta["description"] + assert "Mantle" in delta["description"] def test_g08_feature_status_unknown_blocks_readiness(): @@ -939,6 +1275,7 @@ def test_n03_tool_requirement_never_lands_on_unevidenced_nova(): # land on the evidenced Converse candidate (Claude), never claim Nova native. rec = _recommend( _workload( + source={"model_ids": ["gpt-4"]}, requirements={ "governance": ["guardrails"], "critical_features": ["tool_or_function_calling"], @@ -1000,10 +1337,9 @@ def test_tier_map_luna_source_maps_to_haiku_on_converse(): assert order[0] == "openai_gpt_5_6_luna" assert order[1] == "anthropic_claude_haiku_4_5" -def test_tier_map_terra_and_55_sources_map_to_sonnet_on_converse(): - # Terra (5.6) keeps itself — same-model runtime path exists. GPT-5.5 has NO - # runtime path (mantle-only, re-verified 2026-08-21), so its governance - # Converse target remains the Claude balanced tier. +def test_tier_map_terra_and_55_sources_keep_themselves_on_converse(): + # Terra (5.6) and GPT-5.5 are recognized same-model identities with a cataloged + # runtime_converse path (us./global. CRIS ids), so governance keeps the model. rec_terra = _recommend( _workload(source={"model_ids": ["gpt-5.6-terra"]}, requirements={"governance": ["guardrails"]}) @@ -1013,7 +1349,11 @@ def test_tier_map_terra_and_55_sources_map_to_sonnet_on_converse(): _workload(source={"model_ids": ["gpt-5.5"]}, requirements={"governance": ["guardrails"]}) ) - assert rec_55["primary_model"] == "anthropic.claude-sonnet-5" + assert rec_55["primary_model"] == "openai.gpt-5.5" + assert rec_55["api_path"] == "runtime_converse" + # The Sonnet tier stays the cross-family fallback behind both. + order = oai._converse_candidate_order({"model_ids": ["gpt-5.5"]}) + assert order[:2] == ["openai_gpt_5_5", "anthropic_claude_sonnet_5"] def test_tier_map_falls_back_across_tiers_on_capability_evidence(): # The capability-evidence fallback mechanism: when the first candidate in the @@ -1047,11 +1387,11 @@ def test_same_model_governance_still_recommended_end_to_end(): @pytest.mark.parametrize("surface,path", [ - ("responses", "mantle_openai_responses"), - ("chat_completions", "mantle_openai_chat"), + ("responses", "runtime_openai_responses"), + ("chat_completions", "runtime_openai_chat"), ]) @pytest.mark.parametrize("region", ["us-east-1", "us-west-2"]) -def test_astra_keeps_source_model_and_api_on_mantle(surface, path, region): +def test_astra_keeps_source_model_and_api_on_runtime(surface, path, region): rec = oai.recommend_openai_workload( _workload(source={"model_ids": ["gpt-6-astra"], "api_surface": surface}), region, OPENAI_CATALOG, @@ -1059,6 +1399,7 @@ def test_astra_keeps_source_model_and_api_on_mantle(surface, path, region): assert rec["decision_status"] == "recommended" assert rec["primary_model"] == "openai.gpt-6-astra" assert rec["api_path"] == path + assert rec["mantle_selector"] is None assert rec["source_analysis"]["source_family"] == "reasoning" assert rec["source_analysis"]["version_changed"] is False assert "chat_completions_to_responses_required" not in _codes(rec["blocks"]) @@ -1066,16 +1407,44 @@ def test_astra_keeps_source_model_and_api_on_mantle(surface, path, region): assert "sampling_params_unverified" in _codes(rec["tuning"]) +@pytest.mark.parametrize("surface,path", [ + ("responses", "mantle_openai_responses"), + ("chat_completions", "mantle_openai_chat"), +]) +@pytest.mark.parametrize("region", ["us-east-1", "us-west-2"]) +def test_astra_keeps_source_model_and_api_on_mantle_with_selector(surface, path, region): + rec = oai.recommend_openai_workload( + _workload(source={"model_ids": ["gpt-6-astra"], "api_surface": surface}, + requirements={"mantle_selector": "server_side_tools"}), + region, OPENAI_CATALOG, + ) + assert rec["decision_status"] == "recommended" + assert rec["primary_model"] == "openai.gpt-6-astra" + assert rec["api_path"] == path + assert rec["mantle_selector"] == "server_side_tools" + assert "chat_completions_to_responses_required" not in _codes(rec["blocks"]) + assert "sampling_params_unverified" in _codes(rec["tuning"]) + + @pytest.mark.parametrize("region", ["us-east-2", "eu-west-1"]) @pytest.mark.parametrize("surface", ["responses", "chat_completions"]) def test_astra_mantle_outside_supported_regions_requires_decision(region, surface): rec = oai.recommend_openai_workload( - _workload(source={"model_ids": ["gpt-6-astra"], "api_surface": surface}), + _workload(source={"model_ids": ["gpt-6-astra"], "api_surface": surface}, + requirements={"mantle_selector": "server_side_tools"}), region, OPENAI_CATALOG, ) assert rec["decision_status"] == "decision_required" assert rec["primary_model"] is None assert "mantle_region_unavailable" in _codes(rec["blocks"]) + # Without a selector the bedrock-runtime path keeps Astra in those regions. + runtime = oai.recommend_openai_workload( + _workload(source={"model_ids": ["gpt-6-astra"], "api_surface": surface}), + region, OPENAI_CATALOG, + ) + assert runtime["decision_status"] == "recommended" + assert runtime["primary_model"] == "openai.gpt-6-astra" + assert runtime["api_path"].startswith("runtime_openai_") @pytest.mark.parametrize("surface,preferred,expected,delta", [ @@ -1085,13 +1454,21 @@ def test_astra_mantle_outside_supported_regions_requires_decision(region, surfac ("responses", "mantle_openai_chat", "mantle_openai_chat", "responses_to_chat_request"), ("responses", "mantle_openai_responses", "mantle_openai_responses", None), ("chat_completions", "runtime_converse", "runtime_converse", "openai_sdk_to_converse"), + ("responses", "runtime_openai_responses", "runtime_openai_responses", None), + ("chat_completions", "runtime_openai_chat", "runtime_openai_chat", None), + ("chat_completions", "runtime_openai_responses", "runtime_openai_responses", + "chat_to_responses_request"), + ("responses", "runtime_openai_chat", "runtime_openai_chat", "responses_to_chat_request"), ]) def test_astra_explicit_api_preference_reaches_validated_output(surface, preferred, expected, delta): + requirements = {"preferred_api_path": preferred, "data_residency": "global_allowed"} + if preferred.startswith("mantle_"): + requirements["mantle_selector"] = "server_side_tools" data = { "schema_version": 2, "region": "us-west-2", "primary_unit": "chat-svc", "workloads": [_workload( source={"model_ids": ["gpt-6-astra"], "api_surface": surface}, - requirements={"preferred_api_path": preferred, "data_residency": "global_allowed"}, + requirements=requirements, )], } jsonschema.validate(data, json.loads((SCRIPTS / "schemas/model-recommendation-input.json").read_text())) @@ -1101,19 +1478,29 @@ def test_astra_explicit_api_preference_reaches_validated_output(surface, preferr assert rec["decision_status"] == "recommended" assert rec["api_path"] == expected assert rec["primary_model"] == "openai.gpt-6-astra" + assert rec["mantle_selector"] == ("server_side_tools" if expected.startswith("mantle_") else None) + if expected.startswith("runtime_"): + assert rec["invocation_model_id"] == "global.openai.gpt-6-astra" if delta: assert delta in _delta_codes(rec) assert not any("GPT-5.x" in block["message"] for block in rec["blocks"]) -@pytest.mark.parametrize("preferred,constraints", [ - ("mantle_openai_chat", {"governance": ["guardrails"]}), - ("mantle_openai_responses", {"governance": ["guardrails"]}), - ("runtime_converse", {"api_continuity": "required"}), - ("runtime_invoke", {}), - ("mantle_messages", {}), +@pytest.mark.parametrize("preferred,constraints,block", [ + # A selector plus bedrock-runtime-only governance is the root conflict and is + # reported before the path preference is examined. + ("mantle_openai_chat", {"governance": ["guardrails"], "mantle_selector": "server_side_tools"}, + "model_path_decision_required"), + ("mantle_openai_responses", {"governance": ["guardrails"], "mantle_selector": "server_side_tools"}, + "model_path_decision_required"), + ("mantle_openai_chat", {"governance": ["guardrails"]}, "preferred_api_path_conflict"), + ("mantle_openai_responses", {}, "preferred_api_path_conflict"), # no mantle_selector + ("runtime_converse", {"api_continuity": "required"}, "preferred_api_path_conflict"), + ("runtime_openai_responses", {"governance": ["guardrails"]}, "preferred_api_path_conflict"), + ("runtime_invoke", {}, "preferred_api_path_conflict"), + ("mantle_messages", {}, "preferred_api_path_conflict"), ]) -def test_astra_api_preference_conflicts_require_an_explicit_decision(preferred, constraints): +def test_astra_api_preference_conflicts_require_an_explicit_decision(preferred, constraints, block): rec = oai.recommend_openai_workload( _workload( source={"model_ids": ["gpt-6-astra"]}, @@ -1122,20 +1509,35 @@ def test_astra_api_preference_conflicts_require_an_explicit_decision(preferred, ) assert rec["decision_status"] == "decision_required" assert rec["api_path"] is None + assert block in _codes(rec["blocks"]) + if preferred.startswith("mantle_") and "mantle_selector" not in constraints: + assert "mantle_selector" in rec["blocks"][0]["message"] + + +@pytest.mark.parametrize("preferred", ["mantle_openai_responses", "mantle_openai_chat"]) +def test_mantle_preference_without_selector_fails_closed_for_every_source(preferred): + rec = _recommend(_workload(requirements={"preferred_api_path": preferred})) + assert rec["decision_status"] == "decision_required" assert "preferred_api_path_conflict" in _codes(rec["blocks"]) + assert not any(o["api_path"].startswith("mantle_") for o in rec["decision_options"]) +# Astra caching evidence (FEAT-001 rows): runtime_openai_responses and +# runtime_openai_chat carry prompt_caching; on Mantle only Responses does. The +# invariant: a required cache is checked on the API actually selected. @pytest.mark.parametrize("region", ["us-east-1", "us-west-2"]) -@pytest.mark.parametrize("surface,preferred,expected", [ - ("chat_completions", None, None), - ("responses", "mantle_openai_chat", None), - ("responses", None, "mantle_openai_responses"), - ("chat_completions", "mantle_openai_responses", "mantle_openai_responses"), +@pytest.mark.parametrize("surface,extra,expected", [ + ("chat_completions", {}, "runtime_openai_chat"), + ("responses", {}, "runtime_openai_responses"), + ("chat_completions", {"mantle_selector": "server_side_tools"}, None), + ("responses", {"mantle_selector": "server_side_tools"}, "mantle_openai_responses"), + ("responses", {"preferred_api_path": "mantle_openai_chat", + "mantle_selector": "server_side_tools"}, None), + ("chat_completions", {"preferred_api_path": "mantle_openai_responses", + "mantle_selector": "server_side_tools"}, "mantle_openai_responses"), ]) -def test_astra_required_caching_is_checked_on_the_selected_api(region, surface, preferred, expected): - requirements = {"critical_features": ["prompt_caching"]} - if preferred: - requirements["preferred_api_path"] = preferred +def test_astra_required_caching_is_checked_on_the_selected_api(region, surface, extra, expected): + requirements = {"critical_features": ["prompt_caching"], **extra} rec = oai.recommend_openai_workload( _workload(source={"model_ids": ["gpt-6-astra"], "api_surface": surface}, requirements=requirements), @@ -1155,25 +1557,31 @@ def test_astra_required_caching_is_checked_on_the_selected_api(region, surface, def test_astra_decision_options_ignore_an_explicitly_absent_cache_signal(): workload = _workload( source={"model_ids": ["gpt-6-astra"], "api_surface": "chat_completions"}, - requirements={"preferred_api_path": "mantle_openai_chat"}, + requirements={"preferred_api_path": "mantle_openai_chat", + "mantle_selector": "server_side_tools"}, detected_features=["prompt_caching"], feature_status={"prompt_caching": "absent"}, ) - options = oai._decision_options(OPENAI_CATALOG, workload, "us-west-2") + options = oai._decision_options(OPENAI_CATALOG, workload, "us-west-2", "server_side_tools") assert options[0]["api_path"] == "mantle_openai_chat" + assert options[0]["mantle_selector"] == "server_side_tools" +@pytest.mark.parametrize("extra,path", [ + ({}, "runtime_openai_chat"), + ({"mantle_selector": "server_side_tools"}, "mantle_openai_chat"), +]) @pytest.mark.parametrize("signals", [ {"requirements": {"uses_n": True}}, {"detected_features": ["multiple_candidates_n"]}, ]) -def test_astra_chat_n_requires_verification_without_responses_reshape(signals): - rec = oai.recommend_openai_workload( - _workload(source={"model_ids": ["gpt-6-astra"], "api_surface": "chat_completions"}, - **signals), - "us-west-2", OPENAI_CATALOG, +def test_astra_chat_n_requires_verification_without_responses_reshape(signals, extra, path): + workload = _workload( + source={"model_ids": ["gpt-6-astra"], "api_surface": "chat_completions"}, **signals ) - assert rec["api_path"] == "mantle_openai_chat" + workload["requirements"].update(extra) + rec = oai.recommend_openai_workload(workload, "us-west-2", OPENAI_CATALOG) + assert rec["api_path"] == path assert "multiple_candidates_n_unverified" in _codes(rec["tuning"]) assert "responses_no_n" not in _delta_codes(rec) assert "chat_completions_to_responses_required" not in _codes(rec["blocks"]) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py index 599889f05..0f3ba68e5 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py @@ -242,20 +242,26 @@ def __init__(self, responses=None, chat=None): self.chat = chat or FakeChat() +def _astra_api_client(): + completions = FakeResponses(response={"model": "openai.gpt-6-astra"}) + client = type("ChatClient", (), { + "chat": type("Chat", (), {"completions": completions})(), + "responses": completions, + })() + return completions, client + + @pytest.mark.parametrize("region", ["us-east-1", "us-west-2"]) @pytest.mark.parametrize("surface", ["chat_completions", "responses"]) def test_astra_recommendation_probes_selected_api_in_supported_regions(region, surface): - data = _openai_input() + # Mantle is probed only when a mantle_selector put the workload there. + data = _openai_input({"mantle_selector": "server_side_tools"}) data["region"] = region data["workloads"][0]["source"].update( model_ids=["gpt-6-astra"], api_surface=surface ) recommendation = model_recommendation.recommend(data) - completions = FakeResponses(response={"model": "openai.gpt-6-astra"}) - client = type("ChatClient", (), { - "chat": type("Chat", (), {"completions": completions})(), - "responses": completions, - })() + completions, client = _astra_api_client() result = verify_model_path.verify_recommendation( recommendation, now=NOW, openai_responses_client_factory=lambda region: client, @@ -274,6 +280,35 @@ def test_astra_recommendation_probes_selected_api_in_supported_regions(region, s assert completions.calls[0]["input"] == verify_model_path.PROMPT +@pytest.mark.parametrize("region", ["us-east-1", "us-west-2"]) +@pytest.mark.parametrize("surface", ["chat_completions", "responses"]) +def test_astra_default_recommendation_probes_runtime_api_with_cris_id(region, surface): + # No selector: the default bedrock-runtime OpenAI path is probed through the + # runtime factories with the resolved Global CRIS id; Mantle factories stay idle. + data = _openai_input({"data_residency": "global_allowed"}) + data["region"] = region + data["workloads"][0]["source"].update( + model_ids=["gpt-6-astra"], api_surface=surface + ) + recommendation = model_recommendation.recommend(data) + completions, client = _astra_api_client() + mantle_calls = [] + result = verify_model_path.verify_recommendation( + recommendation, now=NOW, + openai_responses_client_factory=lambda region: mantle_calls.append(region), + openai_chat_client_factory=lambda region: mantle_calls.append(region), + runtime_openai_responses_client_factory=lambda region: client, + runtime_openai_chat_client_factory=lambda region: client, + ) + verification = result["workloads"]["openai-svc"] + assert verification["status"] == "passed" + assert verification["api_path"] == ( + "runtime_openai_chat" if surface == "chat_completions" else "runtime_openai_responses" + ) + assert mantle_calls == [] + assert completions.calls[0]["model"] == "global.openai.gpt-6-astra" + + @pytest.mark.parametrize("prefix", ["us", "global"]) @pytest.mark.parametrize("as_arn", [False, True]) def test_astra_system_profile_representation_reaches_verifier_unchanged(prefix, as_arn): @@ -339,7 +374,9 @@ def _verify_openai(requirements=None, source=None, region="us-east-2", **kwargs) def _opus_recommendation(provider, path, profile): - data = _input() if provider == "anthropic" else _openai_input() + # The OpenAI case uses an unrecognized source id (gpt-4) so governance lands on + # the Claude cross-family Converse tier (Opus under priority quality). + data = _input() if provider == "anthropic" else _openai_input(source={"model_ids": ["gpt-4"]}) data["region"] = "us-east-1" requirements = data["workloads"][0]["requirements"] requirements.update(priority="quality", data_residency="geo_required", @@ -531,7 +568,7 @@ def test_recommendation_cli_drops_orphaned_verification_before_first_write(tmp_p ]) @pytest.mark.parametrize("regions", [("eu-west-2",), ("eu-west-2", "eu-west-1")]) def test_london_application_profile_accepts_in_region_or_matching_geo_destinations(provider, path, regions): - data = _input() if provider == "anthropic" else _openai_input() + data = _input() if provider == "anthropic" else _openai_input(source={"model_ids": ["gpt-4"]}) data["region"] = "eu-west-2" arn = APPLICATION_ARN.replace("us-east-1", "eu-west-2") requirements = data["workloads"][0]["requirements"] @@ -612,10 +649,14 @@ def test_changed_in_region_allowance_invalidates_same_arn_verification(tmp_path) assert "allowed_in_region_model_arns" not in workload["verification"] +# Mantle OpenAI paths are probed only when a mantle_selector put the workload there. +MANTLE_OPENAI = {"api_continuity": "required", "mantle_selector": "background_inference"} + + def test_mantle_responses_probe_calls_responses_create_with_exact_id(): client = FakeOpenAIClient() result = _verify_openai( - {"api_continuity": "required"}, + MANTLE_OPENAI, source={"model_ids": ["gpt-5.6-sol"]}, openai_responses_client_factory=lambda region: client, openai_chat_client_factory=lambda region: client, ) @@ -631,7 +672,7 @@ def test_mantle_responses_probe_calls_responses_create_with_exact_id(): def test_mantle_responses_failure_does_not_substitute_model(): client = FakeOpenAIClient(FakeResponses(error=RuntimeError("access denied"))) result = _verify_openai( - {"api_continuity": "required"}, + MANTLE_OPENAI, source={"model_ids": ["gpt-5.6-sol"]}, openai_responses_client_factory=lambda region: client, openai_chat_client_factory=lambda region: client, ) @@ -643,10 +684,26 @@ def test_mantle_responses_failure_does_not_substitute_model(): assert [c["model"] for c in client.responses.calls] == ["openai.gpt-5.6-sol"] +def test_default_openai_recommendation_probes_runtime_responses_not_mantle(): + client = FakeOpenAIClient(FakeResponses(response={"model": "global.openai.gpt-5.4"})) + mantle_calls = [] + result = _verify_openai( + {"api_continuity": "required", "data_residency": "global_allowed"}, + openai_responses_client_factory=lambda region: mantle_calls.append(region), + runtime_openai_responses_client_factory=lambda region: client, + ) + verification = result["workloads"]["openai-svc"] + + assert verification["status"] == "passed" + assert verification["api_path"] == "runtime_openai_responses" + assert mantle_calls == [] + assert [c["model"] for c in client.responses.calls] == ["global.openai.gpt-5.4"] + + def test_mantle_chat_probe_calls_chat_completions_with_exact_id(): client = FakeOpenAIClient() result = _verify_openai( - {"api_continuity": "required"}, + MANTLE_OPENAI, source={"model_ids": ["gpt-6-sol"], "api_surface": "chat_completions"}, region="us-east-1", openai_chat_client_factory=lambda region: client, @@ -677,7 +734,7 @@ def test_mantle_chat_failure_does_not_substitute_model(): completions = FakeCompletions(error=RuntimeError("access denied")) client = FakeOpenAIClient(chat=FakeChat(completions)) result = _verify_openai( - {"api_continuity": "required"}, + MANTLE_OPENAI, source={"model_ids": ["gpt-6-sol"], "api_surface": "chat_completions"}, region="us-east-1", openai_chat_client_factory=lambda region: client, @@ -708,9 +765,9 @@ def factory(region): # --- bedrock-runtime Responses / Chat Completions / Messages probes ----------- # -# The OpenAI engine does not yet emit its Runtime paths, so these recommendations are -# built by hand from a recommended result and re-pointed at the path under test; this -# keeps every apiPath enum value covered independently of engine defaults. +# These recommendations are built by hand from a recommended result and re-pointed +# at the path under test, so every apiPath enum value stays covered independently +# of the engines' defaults. RECOMMENDATION_SCHEMA = json.loads( (pathlib.Path(verify_model_path.__file__).parent / "schemas" / "model-recommendation.json").read_text() @@ -735,10 +792,25 @@ def _runtime_recommendation(provider, path, path_model_id, invocation_model_id): workload["model_identity"]["path_model_id"] = path_model_id workload["model_identity"]["requires_cris"] = True workload["verification"]["invocation_model_id"] = invocation_model_id + # The output schema requires a string mantle_selector on every mantle_* path. + workload["mantle_selector"] = "workspaces" if path.startswith("mantle_") else None jsonschema.validate(recommendation, RECOMMENDATION_SCHEMA) return recommendation +def test_output_schema_rejects_mantle_path_without_selector(): + recommendation = _runtime_recommendation( + "openai", "mantle_openai_responses", "openai.gpt-5.4", "openai.gpt-5.4" + ) + workload = next(iter(recommendation["workloads"].values())) + workload["mantle_selector"] = None + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(recommendation, RECOMMENDATION_SCHEMA) + del workload["mantle_selector"] + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(recommendation, RECOMMENDATION_SCHEMA) + + def test_runtime_responses_probe_calls_responses_create_with_exact_cris_id(): client = FakeOpenAIClient(FakeResponses(response={"model": "global.openai.gpt-5.4"})) recommendation = _runtime_recommendation( From 6fee2d95729d43ce401ad515fb31f8990a8f623f Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 08:15:55 -0800 Subject: [PATCH 30/41] docs(aws-startup-advisor): agent-advisor model-recommend, handoff and POC docs follow the bedrock-runtime default and carry mantle_selector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring the agent-advisor prose in line with the engines reworked in the two previous commits (bedrock-runtime by default, Mantle only by a recorded mantle_selector from bedrock-endpoint-policy.md): - decision-refs/model-selection.md: list runtime_messages, runtime_openai_chat and runtime_openai_responses; add the "Endpoint policy" principle pointing at references/vendored/ai/bedrock-endpoint-policy.md (reference, not restated); the conflict example is now "a verified Mantle selector plus Converse-only governance". - phases/model-recommend/model-recommend.md: load the vendored policy as _knowledge; Step 2 asks both providers for a verified Mantle-only need (mantle_selector) and for data_residency: in_region_required; the OpenAI paragraph describes the runtime default, the selector rule (explicit, web_search, in_region_processing), GPT-5.5/5.4 same-model recognition, the Converse governance split, the two-runtime-option decision and the same-family newest-first fallback; PR 1's "deliberate consequence" / "ordering limitation" / "does not yet recognize" / "currently reshaped to Responses" disclosures are deleted; Step 7 names the client packages per path (--with anthropic --with aws-bedrock-token-generator for runtime_messages; --with openai --with aws-bedrock-token-generator for the four OpenAI-SDK paths); Step 8 shows mantle_selector. - confirm.md / design.md: model_decision and model_recommendation carry mantle_selector (null on bedrock-runtime paths). - poc.md: client construction for runtime_messages (bedrock-runtime.{region}.amazonaws.com/anthropic, bearer token) and runtime_openai_responses / runtime_openai_chat (bedrock-runtime.{region}.amazonaws.com/openai/v1, CRIS id). - migration-plan.md Step 3.5: translation table advisor api_path -> migration_path + required_client_api (+ mantle_selector); the matches_advisor_model snippet gains _RUNTIME_OPENAI, a proprietary-GPT tuple covering GPT-6 / GPT-5.6 / GPT-5.5 / GPT-5.4 (open-weight gpt-oss keeps the exact comparison, pinned by llm-to-bedrock's test_astra_execution_contract), a runtime_messages branch and a mantle_* branch that requires an equal mantle_selector and rejects a null one (a legacy contract without the field keeps the exact comparison so PR 2's locked comparator tests stay green). - handoff/handoff-migration.md: the summary carries per-unit api_path, invocation_model_id and mantle_selector plus the same translation table. - gcp-to-aws design-ai.md: the TODO(PR 3) comment is replaced by the translation paragraph; no other line touched. - SKILL.md § Files: row for references/vendored/ai/bedrock-endpoint-policy.md (frontmatter description unchanged). - scripts/test_runtime_default_docs.py (new): doc locks for all of the above plus executable checks of the committed comparator for every new path. No catalog, schema, vendored or llm-to-bedrock file changed; no new catalog rows, so no new model-card reads (FEAT-001 cards all read 2026-11-07 under https://docs.aws.amazon.com/bedrock/latest/userguide/: model-card-openai-gpt-56-sol/-56-terra/-56-luna/-55/-54/-6-astra/-6-sol/ -6-luna/-6-1-sol.html, model-card-anthropic-claude-opus-5-5/-sonnet-5/ -haiku-4-5.html). Verification (worktree P=plugins/aws-startup-advisor, 2026-11-08): - cd P/skills/agent-advisor/scripts && uv run --group dev pytest -q -> 719 passed (694 after FEAT-003; +25 new in test_runtime_default_docs.py) - cd P/skills/llm-to-bedrock/scripts && uv run --group dev pytest -q -> 622 passed (unchanged) - cd P && python3 tools/sync-vendored.py --check -> PASS, 65 files - mise run lint:model-staleness -> OK; lint:model-catalog-drift -> OK (3 pre-existing warnings: sonnet-5-5, haiku-5-5, gpt-6.1-terra absent by design, stage 2 of upstream #566); lint:startup-model-pricing -> OK; lint:artifacts -> PASS 99 artifacts, 0 errors; lint:vendored-parity -> PASS; lint:skill-refs -> PASS 1531 refs, 0 missing, 7 baselined; lint:consent-disclosure -> OK - mise run lint:model-id -> fails ONLY on the untracked, git-ignored local P/skills/llm-to-bedrock/scripts/.venv/.../anthropic/_constants.py; on a copy of the tracked tree without that venv -> "model-id lint: OK (19 retired IDs)" - mise run test -> 1430 passed, 222 subtests, 1 failed: tests/test_model_id_lint.py::test_real_plugin_passes, the same local .venv noise; that test file -> 11 passed on the venv-free copy - mise run test:telemetry -> 311 passed; test:heroku-source-review -> pass - markdownlint-cli2 --config ../../.markdownlint-cli2.yaml '**/*.md' '#node_modules' '#**/.venv/**' -> 824 files, 0 errors - root python3 tools/validate.py -> All validations passed; git diff --check -> clean - root mise run build -> exit 1. lint:spec fails only on 6 .venv symlinks from the two untracked local venvs (agent-advisor/scripts/.venv, llm-to-bedrock/scripts/.venv); lint:manifests, lint:skill-parity, test:tools pass. [security] gitleaks ran and reported exactly one finding: the pre-existing upstream commit f8d09fd1 (tests/test_bedrock_migration_assessment_discover.py:118, "claude-3.5-sonnet" matched generic-api-key) - not introduced by this branch, reported distinctly, not fixed here. --- .../skills/agent-advisor/SKILL.md | 1 + .../decision-refs/model-selection.md | 17 +- .../references/handoff/handoff-migration.md | 14 ++ .../references/phases/confirm/confirm.md | 16 +- .../references/phases/design/design.md | 5 +- .../phases/migration-plan/migration-plan.md | 49 ++++- .../phases/model-recommend/model-recommend.md | 96 +++++----- .../references/phases/poc/poc.md | 8 +- .../scripts/test_runtime_default_docs.py | 171 ++++++++++++++++++ .../references/phases/design/design-ai.md | 10 +- 10 files changed, 317 insertions(+), 70 deletions(-) create mode 100644 plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_runtime_default_docs.py diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/SKILL.md b/plugins/aws-startup-advisor/skills/agent-advisor/SKILL.md index aa018c32b..1c084c62a 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/SKILL.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/SKILL.md @@ -161,6 +161,7 @@ Stage 3 reads gcp-to-aws artifacts ONLY via this recorded path, never by re-glob | File | Purpose | | ---------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | | `references/vendored/dsl/INTERPRETER.md` | Vendored DSL execution contract (interpreter loop + gate protocol) | +| `references/vendored/ai/bedrock-endpoint-policy.md` | Vendored Bedrock endpoint policy (bedrock-runtime default, mantle_selector enum) obeyed by Model Recommend | | `references/phases/intake/intake.md` | Entry point + technical background + open context | | `references/phases/discover/discover.md` | Lightweight code detection | | `references/phases/clarify/clarify.md` | Clarify orchestrator + answer mapping to scoring keys | diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/decision-refs/model-selection.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/decision-refs/model-selection.md index 0dbaaa6e3..5542a6365 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/decision-refs/model-selection.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/decision-refs/model-selection.md @@ -20,18 +20,23 @@ The per-workload `(model, api_path)` decision is deterministic and lives in code file — it is dispatched by source provider so the path vocabulary and rules differ per provider: - **Anthropic** → `scripts/anthropic_model_recommendation.py` + `references/vendored/ai/models/anthropic-bedrock.json` - (paths: `mantle_messages`, `runtime_converse`, `runtime_invoke`). + (paths: `mantle_messages`, `runtime_messages`, `runtime_converse`, `runtime_invoke`). - **OpenAI** → `scripts/openai_model_recommendation.py` + `references/vendored/ai/models/openai-bedrock.json` - (paths: `mantle_openai_chat`, `mantle_openai_responses`, `runtime_converse`; preserve Astra's - supported source API and apply each model's dated capability and region checks). + (paths: `mantle_openai_chat`, `mantle_openai_responses`, `runtime_openai_chat`, + `runtime_openai_responses`, `runtime_converse`; preserve the source API on the exact source + model and apply each model's dated capability and region checks). - **Azure OpenAI / other** → generic `provider_module_pending` until a dedicated module exists. Do not restate the per-provider path rules here — the dated catalogs and the engine are authoritative (a table here would drift). The engine applies these provider-neutral principles: -- **Conflicts are not precedence.** When two hard requirements cannot share one path (e.g. API - continuity plus runtime-only governance), emit `decision_required` with both tradeoff options - and rerun after the user resolves the requirement — never silently let one side win. +- **Endpoint policy.** The engine obeys `references/vendored/ai/bedrock-endpoint-policy.md`: + `bedrock-runtime` by default; a `mantle_*` path only with a recorded `mantle_selector` from + that file; API continuity alone never selects Mantle. +- **Conflicts are not precedence.** When two hard requirements cannot share one path (e.g. a + verified Mantle selector plus Converse-only governance), emit `decision_required` with both + tradeoff options and rerun after the user resolves the requirement — never silently let one + side win. - **A dated snapshot is not a permanent claim.** Catalog data is point-in-time — e.g. Sonnet 4.6 was runtime-only in the `2026-06-24` snapshot, and six days later Sonnet 5 launched with full Mantle Messages support, changing the default recommendation. Every recommendation diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/handoff/handoff-migration.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/handoff/handoff-migration.md index b28ed9ed3..dea523fc4 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/handoff/handoff-migration.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/handoff/handoff-migration.md @@ -15,6 +15,20 @@ model + services, coarse model family mapping (the source model from the user's answer → the Bedrock family per `model-selection.md`; no prices), and the rationale (top scoring signals + eliminations, from design.json's `scores`/`eliminated`). +The summary additionally carries, per model-bearing unit, the accepted `api_path`, +`invocation_model_id` and `mantle_selector` from `confirm.json` / `design.json` +(`model_decision` / `model_recommendation`), plus this translation table so that `gcp-to-aws` +Design and `llm-to-bedrock` write `migration_path` / `required_client_api` / `mantle_selector` +without re-deciding the endpoint (the vocabulary is +`references/vendored/ai/bedrock-endpoint-policy.md`): + +- `runtime_openai_responses` → `migration_path: "runtime_openai_cris"`, `required_client_api: "responses"` +- `runtime_openai_chat` → `migration_path: "runtime_openai_cris"`, `required_client_api: "chat_completions"` +- `runtime_converse` on a proprietary GPT model (`openai.gpt-6*`, `openai.gpt-5.6-*`, `openai.gpt-5.5`, `openai.gpt-5.4`) → `migration_path: "runtime_openai_cris"`, `required_client_api: "converse"` +- `runtime_messages` → `migration_path: "runtime_messages"`, `required_client_api: "messages"`, `aws_model_id` = the advisor `invocation_model_id` +- `mantle_openai_responses` / `mantle_openai_chat` / `mantle_messages` → the same `migration_path`, `required_client_api` `responses` / `chat_completions` / `messages`, and `bedrock_models[].mantle_selector` copied verbatim (an advisor Mantle contract without a selector is a mismatch — return to Model Recommend) +- any other path (`runtime_converse` on a Claude or open-weight model, `runtime_invoke`) → `migration_path` equal to the advisor `api_path`, no `mantle_selector` + ## Step 2 — Check downstream availability Both downstream skills — `gcp-to-aws` and `llm-to-bedrock` — are part of this same plugin, so diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/confirm/confirm.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/confirm/confirm.md index 70440200b..46183100a 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/confirm/confirm.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/confirm/confirm.md @@ -27,7 +27,7 @@ _postconditions: _on_failure: _halt_and_inform - _validate_json: confirm.json _on_failure: _halt_and_inform - - _assert: "confirm.json has one units[] entry per agent_session unit (each with its own confirmed deployment_model, agentcore_services, tool_choices, model_decision, and chosen_runtime when that unit's verdict was co_recommend), plus top-level fields mirroring the primary unit; every model-bearing unit's model_decision copies primary_model, api_path, invocation_model_id, and the independent verification_status from model-recommendation.json plus model-verification.json when present unless the user requested changed requirements and the deterministic engine was rerun; accepted means the recommendation strategy was accepted and never implies verification_status == passed; a single-unit run may use the flat top-level shape alone; confirm.json carries resolved_runtimes{: } for every unit (the per-unit rule/Tier-1 picks used by the platform gate); confirm.json records platform_decision {mode, platform} — asked only when the resolved runtimes diverge, silent split otherwise; when the gate was asked, platform_decision.offer records the superset and per-unit sacrifices; every temporal_worker_poll unit's Tier-1 pick (including any user AskUserQuestion choice for Tier-1 rules 1/4) is persisted in resolved_runtimes so Design consumes it verbatim and never re-asks" + - _assert: "confirm.json has one units[] entry per agent_session unit (each with its own confirmed deployment_model, agentcore_services, tool_choices, model_decision, and chosen_runtime when that unit's verdict was co_recommend), plus top-level fields mirroring the primary unit; every model-bearing unit's model_decision copies primary_model, api_path, mantle_selector, invocation_model_id, and the independent verification_status from model-recommendation.json plus model-verification.json when present unless the user requested changed requirements and the deterministic engine was rerun; accepted means the recommendation strategy was accepted and never implies verification_status == passed; a single-unit run may use the flat top-level shape alone; confirm.json carries resolved_runtimes{: } for every unit (the per-unit rule/Tier-1 picks used by the platform gate); confirm.json records platform_decision {mode, platform} — asked only when the resolved runtimes diverge, silent split otherwise; when the gate was asked, platform_decision.offer records the superset and per-unit sacrifices; every temporal_worker_poll unit's Tier-1 pick (including any user AskUserQuestion choice for Tier-1 rules 1/4) is persisted in resolved_runtimes so Design consumes it verbatim and never re-asks" _on_failure: _halt_and_inform --- @@ -141,12 +141,13 @@ null model/path. The user may: - change a requirement. In that case update `model-recommendation-input.json`, rerun `model_recommendation.py`, and present the new result. -Do not directly edit `primary_model`, `api_path`, or `invocation_model_id`, and do not let -runtime scoring or a downstream migration engine replace them. Accepting the strategy and -proving target-account access are independent. Record each accepted result as -`model_decision: {"primary_model": "...", "api_path": "...", "invocation_model_id": "... | -null", "accepted": true, "verification_status": "not_run | passed | failed | -needs_resolution"}` in the matching unit. Use `model-verification.json` when present; +Do not directly edit `primary_model`, `api_path`, `mantle_selector`, or `invocation_model_id`, +and do not let runtime scoring or a downstream migration engine replace them. Accepting the +strategy and proving target-account access are independent. Record each accepted result as +`model_decision: {"primary_model": "...", "api_path": "...", "mantle_selector": "... | null", +"invocation_model_id": "... | null", "accepted": true, "verification_status": "not_run | passed | +failed | needs_resolution"}` in the matching unit (`mantle_selector` is copied verbatim from the +recommendation: a string on `mantle_*` paths, `null` on every bedrock-runtime path). Use `model-verification.json` when present; otherwise use the recommendation's `verification.probe_status`. Model-less units omit `model_decision`. @@ -177,6 +178,7 @@ mirror the primary unit's entry. "model_decision": { "primary_model": "anthropic.claude-sonnet-4-6", "api_path": "runtime_converse", + "mantle_selector": null, "invocation_model_id": "global.anthropic.claude-sonnet-4-6", "verification_status": "passed", "accepted": true diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/design/design.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/design/design.md index 1146e523d..72a60d6bb 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/design/design.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/design/design.md @@ -28,7 +28,7 @@ _postconditions: _on_failure: _halt_and_inform - _assert: "Every unit selecting registry records its intended Registry Region and current-run verified Registry availability; cached or unavailable Regions cannot satisfy this check. A user-confirmed omission is reflected in both confirm.json and design.json service lists, including the primary-unit mirrors." _on_failure: _halt_and_inform - - _assert: "design.json has one units[] entry per inventory unit, a platform block consistent with confirm.platform_decision, and top-level legacy fields mirroring the primary unit; design.json has top-level verdict, chosen_runtime, deployment_model, agentcore_compute_type, agentcore_services, model_recommendation, and carries scores + eliminated (and blocking_constraints when present) copied verbatim from scoring-result.json; every agentcore-verdict unit carries agentcore_compute_type (microvms or instances) verbatim from its scoring result and every non-agentcore unit carries null — the compute type is never re-derived in Design; every model-bearing unit's model_recommendation is derived from the matching model-recommendation.json workload entry and accepted confirm.model_decision, including model_identity, model, api_path, invocation_model_id, source, source_analysis, feature_assessment, compatibility, architecture_impacts, additional_targets (separate-modality target contracts, carried verbatim when present), blocks, tuning, migration_deltas, evaluation, rollout, provisional verification, and live_verification when model-verification.json exists; no model was independently selected from scoring-result.json; handoff_required is true iff ANY unit's effective_runtime needs a compute handoff — one of ecs, eks, fargate, or batch (not just the primary/winning runtime; AgentCore/Lambda/Lambda MicroVMs are self-contained); when temporal units exist, design.json has a temporal block recording the Way, per-queue Tier 1 rule ids, and Serverless Workers labeled Public Preview regardless of any docs label; Workflow orchestration code is never rewritten; every unit carries a key_change line derived from its runtime's service card; every non-agent unit's verdict equals the runtime its workload-classes rule maps to (W1→eks/ecs; W2→batch; W3/W4→lambda; W5/W6→fargate) — verdict and workload_class are never contradictory; every unit carries an effective_runtime equal to platform.runtime when platform.mode is consolidated, else its own resolved runtime (a co_recommend unit resolves to its confirm chosen_runtime) — effective_runtime is always a concrete runtime enum, never the literal co_recommend" + - _assert: "design.json has one units[] entry per inventory unit, a platform block consistent with confirm.platform_decision, and top-level legacy fields mirroring the primary unit; design.json has top-level verdict, chosen_runtime, deployment_model, agentcore_compute_type, agentcore_services, model_recommendation, and carries scores + eliminated (and blocking_constraints when present) copied verbatim from scoring-result.json; every agentcore-verdict unit carries agentcore_compute_type (microvms or instances) verbatim from its scoring result and every non-agentcore unit carries null — the compute type is never re-derived in Design; every model-bearing unit's model_recommendation is derived from the matching model-recommendation.json workload entry and accepted confirm.model_decision, including model_identity, model, api_path, mantle_selector, invocation_model_id, source, source_analysis, feature_assessment, compatibility, architecture_impacts, additional_targets (separate-modality target contracts, carried verbatim when present), blocks, tuning, migration_deltas, evaluation, rollout, provisional verification, and live_verification when model-verification.json exists; no model was independently selected from scoring-result.json; handoff_required is true iff ANY unit's effective_runtime needs a compute handoff — one of ecs, eks, fargate, or batch (not just the primary/winning runtime; AgentCore/Lambda/Lambda MicroVMs are self-contained); when temporal units exist, design.json has a temporal block recording the Way, per-queue Tier 1 rule ids, and Serverless Workers labeled Public Preview regardless of any docs label; Workflow orchestration code is never rewritten; every unit carries a key_change line derived from its runtime's service card; every non-agent unit's verdict equals the runtime its workload-classes rule maps to (W1→eks/ecs; W2→batch; W3/W4→lambda; W5/W6→fargate) — verdict and workload_class are never contradictory; every unit carries an effective_runtime equal to platform.runtime when platform.mode is consolidated, else its own resolved runtime (a co_recommend unit resolves to its confirm chosen_runtime) — effective_runtime is always a concrete runtime enum, never the literal co_recommend" _on_failure: _halt_and_inform --- @@ -194,7 +194,8 @@ Assemble per unit: Each agent unit's confirmed runtime/services are independent — never copy the primary unit's. Write the compatibility `model_recommendation` object as `{"model": primary_model, "model_identity": ..., "reasoning": rationale joined for display, - "api_path": api_path, "invocation_model_id": ..., "source": ..., "source_analysis": ..., + "api_path": api_path, "mantle_selector": ... (verbatim; null on bedrock-runtime paths), + "invocation_model_id": ..., "source": ..., "source_analysis": ..., "feature_assessment": ..., "alternatives": ..., "compatibility": ..., "architecture_impacts": ..., "additional_targets": ... (copy verbatim when the recommendation has them; a separate-modality target with status unresolved must NOT be turned into a runnable diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/migration-plan/migration-plan.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/migration-plan/migration-plan.md index cb0ce6876..edf810c0d 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/migration-plan/migration-plan.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/migration-plan/migration-plan.md @@ -356,34 +356,65 @@ are not modified and gcp never sees this step: For every matched, model-bearing unit, compare the plan's model and migration path with that unit's `design.json` model contract, including its separate `invocation_model_id`. Carry the injected invocation identifier into the corresponding GCP `aws_model_id` unchanged. -For a proprietary GPT `runtime_converse` decision, GCP represents the same API as -`migration_path: "runtime_openai_cris"` with `required_client_api: "converse"` on the matched model row. +GCP represents each advisor `api_path` with the `migration_path` / `required_client_api` +vocabulary of `references/vendored/ai/bedrock-endpoint-policy.md` on the matched model row: + +| Advisor `api_path` | GCP `migration_path` | `required_client_api` | Notes | +| --- | --- | --- | --- | +| `runtime_openai_responses` | `runtime_openai_cris` | `responses` | `aws_model_id` = advisor `invocation_model_id` (CRIS id) | +| `runtime_openai_chat` | `runtime_openai_cris` | `chat_completions` | same | +| `runtime_converse` on a proprietary GPT model (`openai.gpt-6*`, `openai.gpt-5.6-*`, `openai.gpt-5.5`, `openai.gpt-5.4`) | `runtime_openai_cris` | `converse` | same; `runtime_converse` on a Claude or open-weight model keeps the exact comparison | +| `runtime_messages` | `runtime_messages` | `messages` | `aws_model_id` = advisor `invocation_model_id` | +| `mantle_openai_responses` / `mantle_openai_chat` / `mantle_messages` | the same path name | `responses` / `chat_completions` / `messages` | `bedrock_models[].mantle_selector` copied verbatim from the advisor contract; an advisor Mantle contract whose `mantle_selector` is null is a mismatch (only a legacy contract written before the field existed keeps the exact model/path comparison) | + An absent `runtime_api` on a legacy runtime plan means Converse; an explicit Chat/Responses choice is not equivalent. Do not replace a confirmed Global profile with US Geo or strip an ARN. Read each matched model row's `migration_path` and `required_client_api`; use legacy `runtime_api` and the `code_migration` defaults only for a single-unit legacy plan. A secondary unit must not inherit the primary's API or invocation profile. Missing per-unit data requires re-running Design with the injection. -Pass the resolved per-model API to `runtime_api` below. A current Design plan with no -explicit API defaults to Responses and cannot satisfy an advisor Converse decision. +Pass the resolved per-model API to `runtime_api` below and the row's `mantle_selector` (when +present) to `mantle_selector`. A current Design plan with no explicit API defaults to Responses +and cannot satisfy an advisor Converse decision. Use this comparison for each unit (including the single-unit case). It separates logical model identity from the invocation identifier while preserving the existing exact comparison for other paths: ```python -def matches_advisor_model(advisor, plan_model, migration_path, runtime_api=None): +_RUNTIME_OPENAI = {"runtime_openai_responses": "responses", + "runtime_openai_chat": "chat_completions", + "runtime_converse": "converse"} +# Proprietary GPT models reach bedrock-runtime through CRIS ids (open-weight gpt-oss keeps +# the exact comparison below). +_PROPRIETARY_GPT = ("openai.gpt-6", "openai.gpt-5.6-", "openai.gpt-5.5", "openai.gpt-5.4") + + +def matches_advisor_model(advisor, plan_model, migration_path, runtime_api=None, + mantle_selector=None): model = advisor["model"] - proprietary_runtime = model.startswith(("openai.gpt-6", "openai.gpt-5.6-")) - if advisor["api_path"] == "runtime_converse" and proprietary_runtime: - invocation = advisor.get("invocation_model_id") + path = advisor["api_path"] + proprietary = model.startswith(_PROPRIETARY_GPT) + invocation = advisor.get("invocation_model_id") + if path in _RUNTIME_OPENAI and proprietary: if not invocation: return False profile_id = invocation.rsplit("/", 1)[-1] logical_model = (profile_id.split(".", 1)[1] if profile_id.startswith(("us.", "in.", "global.")) else profile_id) + resolved_api = runtime_api or ("converse" if path == "runtime_converse" else None) return (logical_model == model and plan_model == invocation and migration_path == "runtime_openai_cris" - and (runtime_api or "converse") == "converse") + and resolved_api == _RUNTIME_OPENAI[path]) + if path == "runtime_messages": + return (bool(invocation) and plan_model == invocation + and migration_path == "runtime_messages" + and (runtime_api or "messages") == "messages") + if path.startswith("mantle_"): + selector = advisor.get("mantle_selector") + if "mantle_selector" in advisor and selector is None: + return False # a Mantle contract without a selector never validates + return (plan_model == model and migration_path == path + and mantle_selector == selector) return plan_model == model and migration_path == advisor["api_path"] ``` diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md index 013ceaf50..e34ff1353 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md @@ -9,6 +9,7 @@ _input: _knowledge: - { file: references/vendored/ai/models/anthropic-bedrock.json } - { file: references/vendored/ai/models/openai-bedrock.json } + - { file: references/vendored/ai/bedrock-endpoint-policy.md } _assemble: _file: phases/model-recommend/model-recommend-assemble.md _produces: @@ -79,7 +80,14 @@ code or prior answers: - require a provider-native Bedrock request body? - minimum context window and expected output-token ceiling? - use thinking? -- allow Global CRIS, or require a geography-scoped CRIS profile? +- a verified Mantle-only need? — background inference, server-side / pre-configured tools + including Web Search, customer Projects or Workspaces, a model or API listed on + `bedrock-mantle` only, or strict in-Region processing where `bedrock-runtime` offers only a + CRIS profile: record `mantle_selector` with the exact value from + `references/vendored/ai/bedrock-endpoint-policy.md`; otherwise leave it absent — preserving + the Responses, Chat Completions or Messages API is not a selector +- allow Global CRIS, require a geography-scoped profile, or require strict in-Region processing + (`data_residency: in_region_required`)? For OpenAI and Azure OpenAI sources, ask instead for the provider-neutral requirements (never reuse the Anthropic `preserve_messages_api` switch): @@ -89,39 +97,35 @@ reuse the Anthropic `preserve_messages_api` switch): - require Bedrock Guardrails, invocation logging, or a shared multi-model Converse surface? - does the code use `n` (multiple candidates), hosted state (Assistants/Threads, `previous_response_id`), hosted web/file search, Files API, or vector stores? +- a verified Mantle-only need? — background inference, server-side / pre-configured tools + including Web Search, customer Projects or Workspaces, a model or API listed on + `bedrock-mantle` only, or strict in-Region processing where `bedrock-runtime` offers only a + CRIS profile: record `mantle_selector` with the exact value from + `references/vendored/ai/bedrock-endpoint-policy.md`; otherwise leave it absent — preserving + the Responses, Chat Completions or Messages API is not a selector - minimum context window and expected output-token ceiling? -- allow Global CRIS or require a geography-scoped profile (runtime Converse only)? - -OpenAI is handled by a dedicated provider module (`openai-bedrock.json` catalog). -Recognized GPT-5.6 and GPT-6 source identities are evaluated against their exact same-model -catalog entry before capability/numeric fallbacks. The engine routes only recognized GPT-6 Chat -Completions sources to `mantle_openai_chat`; a GPT-5.x Chat Completions source is currently -reshaped to Responses. That reshape is a current engine limitation, not a model fact: the catalog -records Chat Completions as supported on Mantle for GPT-5.5 and GPT-5.4 (model cards read -2026-11-06) and as unchecked for GPT-5.6 Sol. GPT-6 Chat Completions sources with required or -preferred API continuity keep that surface through `mantle_openai_chat`; GPT-6 Responses sources -use `mantle_openai_responses`. Both Mantle paths are -in-region only and enforce the selected model's cataloged `region_availability`: Astra reaches -us-east-1 and us-west-2, while Sol, Luna, and 6.1 Sol reach us-east-1 only. Outside that footprint, -the engine skips the same-model Mantle candidate and evaluates its existing capability/numeric -fallbacks. Those fallbacks walk candidates in catalog order, and GPT-5.5 and GPT-5.4 (both -cataloged with Chat Completions on Mantle; GPT-5.5 reaches us-east-1/us-east-2, GPT-5.4 reaches -us-east-1/us-east-2/us-west-2 and us-gov-west-1) precede GPT-6 Astra in that order. A GPT-6 Sol or -6.1 Sol Chat Completions source outside us-east-1 therefore lands on GPT-5.5 in us-east-2 or GPT-5.4 -in us-west-2 through `mantle_openai_chat`, not on Astra. That is a current engine ordering -limitation (a newer-generation source falls back to an older-generation model); treat such a -recommendation as a prompt for the Chat-vs-Converse decision, not as a model endorsement. -Governance requirements select the exact same GPT-5.6 or GPT-6 model through -`runtime_converse` only when the requested Global or geography-scoped CRIS profile is cataloged -for that model and source region; an unsupported profile falls through to an eligible documented -runtime candidate or remains an explicit decision. GPT-5.5/5.4 also carry a cataloged -`runtime_converse` path via `us.`/`global.` CRIS ids (model cards read 2026-11-06); the engine does -not yet recognize them as same-model source identities, so a GPT-5.5/5.4 source under governance -requirements still lands on the Claude Converse tier until that recognition is added. GPT-6 Astra's -runtime surface lacks `Invoke` support (Responses/Chat Completions/Converse only), while -Sol/Luna/6.1 Sol support all four APIs. Every selected path remains provisional until it is probed -in the target account and region. Azure OpenAI remains an explicit `provider_module_pending` -generic result. +- allow Global CRIS, require a geography-scoped profile, or require strict in-Region processing + (`data_residency: in_region_required`)? (any bedrock-runtime path) + +OpenAI is handled by a dedicated provider module (`openai-bedrock.json` catalog). The module +defaults to the bedrock-runtime path that preserves the source API — `runtime_openai_responses` +for a Responses source, `runtime_openai_chat` for a Chat Completions source — on the exact source +model; GPT-5.6, GPT-5.5, GPT-5.4 and GPT-6 identities are all recognized and evaluated against +their same-model catalog entry before capability/numeric fallbacks. A `mantle_openai_*` path is +selected only with a `mantle_selector`: explicit, or derived from a detected `web_search` tool +(the built-in Web Search tool is served on `bedrock-mantle` only) or from +`data_residency: in_region_required` on a runtime model that is CRIS-only in the region. Guardrails +or a shared multi-model Converse surface select `runtime_converse` (same model first, Claude tier +as fallback); invocation logging or CloudWatch alone do not change the path. Required API +continuity plus Guardrails is a `decision_required` between two bedrock-runtime options +(`runtime_openai_` and `runtime_converse`); Mantle is never offered without a selector. +Fallbacks walk same-family candidates first, newest version first, so a GPT-6 Chat Completions +source never falls to GPT-5.4 while Astra or GPT-5.5 is available in the region. Governance +selects the exact same GPT model through `runtime_converse` only when the requested Global or +geography-scoped CRIS profile is cataloged for that model and source region; an unsupported +profile falls through to an eligible documented runtime candidate or remains an explicit +decision. Every selected path remains provisional until it is probed in the target account and +region. Azure OpenAI remains an explicit `provider_module_pending` generic result. For Astra, preserve a compatible explicit API preference before the source default. Keep Astra selected and require a decision when its path, region or feature evidence @@ -130,20 +134,22 @@ cannot satisfy the request; do not silently substitute another model. **OpenRouter/LiteLLM-sourced OpenAI models still use the OpenAI module.** Discover records the underlying `provider` (e.g. `openai`) even when the calls transit a gateway — see `discover.md`'s gateway-detection rule. Do not treat a gateway-routed source as `unknown` or route it to the -generic branch: the model itself, and therefore the Mantle/Converse recommendation, is unaffected +generic branch: the model itself, and therefore the model/path recommendation, is unaffected by the gateway. What changes is `api_continuity`: a gateway-routed source is not, by construction, calling the OpenAI SDK directly, so its actual migration effort is closer to `preferred` than `required` — ask this explicitly rather than assuming `required` from the SDK detection default, -and note in `[TUNE]`/rationale that landing on Mantle from OpenRouter means a real base-URL, -credential, and model-ID-format change, not the zero-code-change claim that applies to a direct -OpenAI SDK caller. +and note in `[TUNE]`/rationale that landing on bedrock-runtime `/openai/v1` (or Mantle) from +OpenRouter means a real base-URL, credential, and model-ID-format change, not the +zero-code-change claim that applies to a direct OpenAI SDK caller. Do not ask users to choose an API path by name unless they already expressed a preference. The deterministic engine ranks `(model, api_path)` candidates together after filtering hard constraints. If an explicit preference exists, record `preferred_api_path`. Record `min_context_tokens`, `expected_output_tokens`, `thinking_enabled`, `data_residency`, -`cris_geography`, and an explicit `inference_profile_id` when known. Do not manufacture a -Global or geography prefix when residency is unresolved. +`cris_geography`, and an explicit `inference_profile_id` when known. Record `mantle_selector` +only when the user confirmed the need. Do not manufacture a Global or geography prefix when +residency is unresolved; with `in_region_required` the runtime invocation id stays null unless +an in-Region model form or an explicit profile applies. ## Step 3 — Record migration-sensitive features @@ -283,8 +289,12 @@ uv run --with boto3 --with anthropic python "$SCRIPTS/verify_model_path.py" \ --output "$RUN_DIR/model-verification.json" ``` -**Each API path needs its own client library.** The line above covers the Anthropic and runtime -paths. A workload on `mantle_openai_responses` or `mantle_openai_chat` additionally needs +**Each API path needs its own client library.** The line above covers `mantle_messages`, +`runtime_converse` and `runtime_invoke`. A workload on `runtime_messages` additionally needs +`--with anthropic --with aws-bedrock-token-generator` (the bearer token for the +`bedrock-runtime.{region}.amazonaws.com/anthropic` base URL). A workload on +`runtime_openai_responses`, `runtime_openai_chat`, `mantle_openai_responses` or +`mantle_openai_chat` additionally needs `--with openai --with aws-bedrock-token-generator`; without them that workload comes back `failed` with a `RuntimeError` naming the missing package, while every other workload still probes normally. So a `failed` status here can mean "the dependency was absent", not "the model @@ -313,8 +323,8 @@ by this rule. Cross-region application profiles still need the reference-profile ## Step 8 — Present the recommendation and advance -For each workload, show the primary model, API path, resolved invocation ID or CRIS TODO, -compatibility groups, architecture impacts, any `additional_targets` (separate-modality targets +For each workload, show the primary model, API path with its `mantle_selector` (`null` on every +bedrock-runtime path), resolved invocation ID or CRIS TODO, compatibility groups, architecture impacts, any `additional_targets` (separate-modality targets such as embeddings/images/audio, shown as unresolved with their named service — never as a runnable model ID), path rationale, `[BLOCKS]`, `[TUNE]`, evaluation mode, rollout gate, and verification status. Do not resolve blockers in chat by silently changing the output. Confirm diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/poc/poc.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/poc/poc.md index 3299bf8ad..9bdd6dfd7 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/poc/poc.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/poc/poc.md @@ -463,14 +463,18 @@ calls the recommended Bedrock model and answers the AgentCore entrypoint contrac `AnthropicBedrockMantle` Messages client. For `runtime_converse`, use the boto3 Converse template. For `runtime_invoke`, use boto3 InvokeModel with the provider-native body. The `mantle_openai_chat` and `mantle_openai_responses` paths use their matching OpenAI-compatible -clients. Never branch on one run-wide `migration_path` string. +clients. For `runtime_messages`, generate an Anthropic SDK client with +`base_url="https://bedrock-runtime.{region}.amazonaws.com/anthropic"` and a Bedrock bearer token +(string `api_key`). For `runtime_openai_responses` / `runtime_openai_chat`, use the OpenAI SDK +with `base_url="https://bedrock-runtime.{region}.amazonaws.com/openai/v1"` and the CRIS +invocation id. Never branch on one run-wide `migration_path` string. The template below applies only to `runtime_converse`. Use it verbatim for that path, substituting `` with the Step 2 invocation ID **for this unit** (or the `TODO: verify model id` placeholder if the MCP was not called). For the other API paths, keep the same `/invocations` and `/ping` HTTP wrapper but replace the imports, client construction, and `run_prompt` implementation with the selected path's client. Do not paste a Converse call -into a Mantle or InvokeModel POC. In a multi-unit POC, each unit's agent.py uses THAT unit's +into a Mantle, InvokeModel or Messages/OpenAI-SDK POC. In a multi-unit POC, each unit's agent.py uses THAT unit's model decision from `design.json.units[].model_recommendation`: ```python diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_runtime_default_docs.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_runtime_default_docs.py new file mode 100644 index 000000000..f3f98c810 --- /dev/null +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_runtime_default_docs.py @@ -0,0 +1,171 @@ +"""Doc lock for the bedrock-runtime default (PR 3 of the endpoint-policy stack). + +The engines (anthropic_model_recommendation.py, openai_model_recommendation.py) default to +bedrock-runtime and select a Mantle path only with a recorded mantle_selector. These tests pin +the agent-advisor prose to that behavior: the Model Recommend questionnaire asks for the +selector and in-Region residency, the decision ref points at the vendored endpoint policy +instead of restating it, the advisor->migration translation emits PR 2's migration_path / +required_client_api / mantle_selector vocabulary for every new path, and the stale PR 1 +disclosures about catalog-order fallbacks and unrecognized GPT-5.5/5.4 identities are gone. +Matching is whitespace-normalized (markdown wraps lines). +""" +import pathlib +import re + +import pytest + +SKILL = pathlib.Path(__file__).parent.parent +PLUGIN_SKILLS = SKILL.parent +MODEL_RECOMMEND_MD = SKILL / "references" / "phases" / "model-recommend" / "model-recommend.md" +MODEL_SELECTION_MD = SKILL / "references" / "decision-refs" / "model-selection.md" +MIGRATION_PLAN_MD = SKILL / "references" / "phases" / "migration-plan" / "migration-plan.md" +HANDOFF_MD = SKILL / "references" / "handoff" / "handoff-migration.md" +CONFIRM_MD = SKILL / "references" / "phases" / "confirm" / "confirm.md" +DESIGN_MD = SKILL / "references" / "phases" / "design" / "design.md" +POC_MD = SKILL / "references" / "phases" / "poc" / "poc.md" +SKILL_MD = SKILL / "SKILL.md" +GCP_DESIGN_AI_MD = PLUGIN_SKILLS / "gcp-to-aws" / "references" / "phases" / "design" / "design-ai.md" + +POLICY_REF = "bedrock-endpoint-policy.md" +NEW_PATHS = ("runtime_openai_responses", "runtime_openai_chat", "runtime_messages") +STALE_PHRASES = ( + "deliberate consequence", + "ordering limitation", + "does not yet recognize", + "currently reshaped to Responses", +) + + +def _norm(path): + return re.sub(r"\s+", " ", path.read_text()) + + +@pytest.mark.parametrize("path", [MODEL_RECOMMEND_MD, MODEL_SELECTION_MD]) +def test_model_docs_reference_the_vendored_endpoint_policy(path): + assert POLICY_REF in _norm(path), f"{path.name} must point at the vendored endpoint policy" + + +def test_model_recommend_asks_for_selector_and_in_region_residency(): + text = _norm(MODEL_RECOMMEND_MD) + for needle in ("mantle_selector", "in_region_required", *NEW_PATHS): + assert needle in text, f"model-recommend.md must name {needle}" + frontmatter = text.split("---", 2)[1] + assert "references/vendored/ai/bedrock-endpoint-policy.md" in frontmatter, \ + "model-recommend.md must load the vendored endpoint policy as knowledge" + + +@pytest.mark.parametrize("phrase", STALE_PHRASES) +def test_model_recommend_drops_pr1_fallback_disclosures(phrase): + assert phrase not in _norm(MODEL_RECOMMEND_MD), \ + f"model-recommend.md still carries the stale disclosure {phrase!r}" + + +def test_model_selection_lists_the_runtime_paths_and_the_selector_rule(): + text = _norm(MODEL_SELECTION_MD) + for needle in (*NEW_PATHS, "mantle_selector", "runtime_converse", "runtime_invoke", + "mantle_messages", "mantle_openai_chat", "mantle_openai_responses"): + assert f"`{needle}`" in text, f"model-selection.md must list {needle}" + assert "verified Mantle selector plus Converse-only governance" in text + assert "API continuity plus runtime-only governance" not in text + + +@pytest.mark.parametrize("path", [MIGRATION_PLAN_MD, HANDOFF_MD, GCP_DESIGN_AI_MD]) +def test_handoff_translation_names_every_new_path_and_the_selector(path): + text = _norm(path) + for needle in (*NEW_PATHS, "mantle_selector", "runtime_openai_cris", "required_client_api"): + assert needle in text, f"{path.name} must translate {needle}" + for api in ("responses", "chat_completions", "messages", "converse"): + assert f"`{api}`" in text or f'"{api}"' in text, f"{path.name} must name required_client_api {api}" + + +def test_migration_plan_snippet_translates_runtime_and_mantle_paths(): + text = _norm(MIGRATION_PLAN_MD) + assert "_RUNTIME_OPENAI" in text + assert "_PROPRIETARY_GPT" in text + for model in ("openai.gpt-6", "openai.gpt-5.6-", "openai.gpt-5.5", "openai.gpt-5.4"): + assert f'"{model}"' in text, f"the proprietary GPT tuple must cover {model}" + assert 'migration_path == "runtime_messages"' in text + assert 'path.startswith("mantle_")' in text + assert "mantle_selector == selector" in text + assert "mantle_selector=None" in text + + +def _comparator(): + section = MIGRATION_PLAN_MD.read_text().split("### Step 3.5", 1)[1].split("### Phase D", 1)[0] + code = re.search(r"```python\n(.*?)\n```", section, re.S).group(1) + ns = {} + exec(compile(code, str(MIGRATION_PLAN_MD), "exec"), ns) # nosec B102 - committed handoff algorithm + return ns["matches_advisor_model"] + + +@pytest.mark.parametrize("path,api,model,invocation", [ + ("runtime_openai_responses", "responses", "openai.gpt-5.4", "global.openai.gpt-5.4"), + ("runtime_openai_chat", "chat_completions", "openai.gpt-5.5", "us.openai.gpt-5.5"), + ("runtime_converse", "converse", "openai.gpt-5.4", "us.openai.gpt-5.4"), + ("runtime_converse", "converse", "openai.gpt-6-astra", "global.openai.gpt-6-astra"), +]) +def test_comparator_translates_runtime_openai_paths_to_runtime_openai_cris(path, api, model, invocation): + matches = _comparator() + advisor = {"model": model, "api_path": path, "invocation_model_id": invocation, + "mantle_selector": None} + assert matches(advisor, invocation, "runtime_openai_cris", api) + other = {"responses", "chat_completions", "converse"} - {api} + for wrong_api in other: + assert not matches(advisor, invocation, "runtime_openai_cris", wrong_api) + assert not matches(advisor, model, "runtime_openai_cris", api) # bare id is not the CRIS id + assert not matches(advisor, invocation, path, api) # advisor path names are not GCP paths + assert not matches({**advisor, "invocation_model_id": None}, invocation, "runtime_openai_cris", api) + + +def test_comparator_translates_runtime_messages(): + matches = _comparator() + advisor = {"model": "anthropic.claude-sonnet-5", "api_path": "runtime_messages", + "invocation_model_id": "global.anthropic.claude-sonnet-5", "mantle_selector": None} + assert matches(advisor, "global.anthropic.claude-sonnet-5", "runtime_messages") + assert matches(advisor, "global.anthropic.claude-sonnet-5", "runtime_messages", "messages") + assert not matches(advisor, "global.anthropic.claude-sonnet-5", "runtime_messages", "converse") + assert not matches(advisor, "anthropic.claude-sonnet-5", "runtime_messages", "messages") + assert not matches(advisor, "global.anthropic.claude-sonnet-5", "converse", "converse") + assert not matches({**advisor, "invocation_model_id": None}, + "global.anthropic.claude-sonnet-5", "runtime_messages", "messages") + + +@pytest.mark.parametrize("path,model", [ + ("mantle_openai_responses", "openai.gpt-5.6-sol"), + ("mantle_openai_chat", "openai.gpt-6-astra"), + ("mantle_messages", "anthropic.claude-opus-5-5"), +]) +def test_comparator_requires_an_equal_non_null_mantle_selector(path, model): + matches = _comparator() + advisor = {"model": model, "api_path": path, "invocation_model_id": model, + "mantle_selector": "background_inference"} + assert matches(advisor, model, path, None, "background_inference") + assert not matches(advisor, model, path) # plan row without a selector + assert not matches(advisor, model, path, None, "web_search") + assert not matches(advisor, model, "runtime_openai_cris", None, "background_inference") + # an advisor Mantle contract that carries a null selector never validates + assert not matches({**advisor, "mantle_selector": None}, model, path, None, "background_inference") + assert not matches({**advisor, "mantle_selector": None}, model, path) + + +def test_gcp_design_ai_todo_is_resolved(): + assert "TODO(PR 3" not in GCP_DESIGN_AI_MD.read_text() + + +def test_poc_constructs_runtime_messages_and_runtime_openai_clients(): + text = _norm(POC_MD) + for needle in NEW_PATHS: + assert f"`{needle}`" in text, f"poc.md must branch on {needle}" + assert "bedrock-runtime.{region}.amazonaws.com/anthropic" in text + assert "bedrock-runtime.{region}.amazonaws.com/openai/v1" in text + + +@pytest.mark.parametrize("path", [CONFIRM_MD, DESIGN_MD]) +def test_confirm_and_design_carry_mantle_selector(path): + assert "mantle_selector" in _norm(path), f"{path.name} must carry mantle_selector" + + +def test_skill_md_files_table_lists_the_vendored_policy(): + text = SKILL_MD.read_text() + files_section = text.split("## Files", 1)[1] + assert "`references/vendored/ai/bedrock-endpoint-policy.md`" in files_section diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index 651520741..d851f9ca9 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -502,7 +502,15 @@ do not manufacture a profile from the region. Keep each unit's model/API/profile For multiple advisor units, write `migration_path` and `required_client_api` on each corresponding `bedrock_models[]` row with its `aws_model_id`; `code_migration` remains the primary-unit default. - +Translate every other advisor `api_path` the same way: `runtime_openai_responses` → +`migration_path: "runtime_openai_cris"` + `required_client_api: "responses"`; `runtime_openai_chat` +→ `runtime_openai_cris` + `"chat_completions"`; `runtime_messages` → +`migration_path: "runtime_messages"` + `required_client_api: "messages"` (keep the advisor +`invocation_model_id` as `aws_model_id`); `mantle_openai_responses` / `mantle_openai_chat` / +`mantle_messages` → the same `migration_path` with `required_client_api` `responses` / +`chat_completions` / `messages` and `bedrock_models[].mantle_selector` copied verbatim from the +advisor contract. An advisor `mantle_*` contract without a `mantle_selector` is a mismatch — +return to Model Recommend rather than inventing one. **Throughput:** load `references/vendored/ai/ai-migration-guardrails.md` and apply the selected model/endpoint's quota rules. Astra runtime uses input TPM plus 10 times output TPM. From 56f927c01ba94d7fe1b35e490c0ee968d6985e61 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 15:03:53 -0800 Subject: [PATCH 31/41] chore(aws-startup-advisor): re-sync the agent-advisor vendored bedrock-endpoint-policy after rebasing onto PR 585 Rebasing fix/agent-advisor-runtime-default onto e6795f32 replayed 87d152b7's copy of skills/shared/ai/bedrock-endpoint-policy.md into skills/agent-advisor/references/vendored/ai/ without the "How a selector is learned" section PR 585 added to the shared source. `python3 tools/sync-vendored.py` restored the byte-identical copy. Verification: `python3 tools/sync-vendored.py --check` -> PASS (65 vendored files byte-identical, README tables match). --- .../vendored/ai/bedrock-endpoint-policy.md | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-endpoint-policy.md index fdc4948a2..6110b3298 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-endpoint-policy.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-endpoint-policy.md @@ -51,6 +51,55 @@ defect: rewrite the row as the `bedrock-runtime` path (`runtime_openai_cris` wit app's `required_client_api`, or `runtime_messages`) or justify it with one of the values above. A `runtime_*` or `converse` row never carries `mantle_selector`. +## How a selector is learned + +The seven values are learned in three different ways; no phase may guess one it cannot learn. + +- **Detected in code, confirmed by the user.** `background_inference`, `web_search`, + `server_side_tools`, `projects` and `workspaces` are the only code-detectable selectors. + Discover's app-code scanners record each match with file:line evidence — as + `workloads[].mantle_signals[]` in the gcp-to-aws / azure-to-aws profile + (`discover-app-code.md` Step 5C) and as top-level `mantle_signals[]` in the provider-neutral + profile (bedrock-migration-assessment `discover-app-code.md` Step 4.5); the usage-API discovery + files add `projects` / `workspaces` / `web_search` signals from the account's own scoping and + model strings. Clarify shows the detected signals and the user confirms or drops each one into + `ai_constraints.mantle_requirements` (full flow Q22b, AI-only Q10.5); when no code was + scanned, Clarify asks the same five outright. A detected signal is a prompt for the question, + never the answer. Design then derives `bedrock_models[].mantle_selector` from that field alone + and records `mantle_requirement` on every Mantle row and `endpoint_assumption` on every + runtime row (`schema-design-aws-ai.md`). + - `background_inference` — positive: `background=True` / `background: true` on + `responses.create`, or a raw `"background": true` Responses body. Not a signal: `store=True`, + `stream=True`, or polling `responses.retrieve` without `background`. `bedrock-runtime` + rejects `background=true` with a 400 + (https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, read + 2026-11-07). + - `web_search` — positive: `tools=[{"type": "web_search"}]`, `"type": "web_search_preview"` + (any dated `web_search_preview_*` form), Anthropic `"type": "web_search_20250305"`, + OpenRouter `plugins: [{"id": "web"}]` or a `:online` model suffix. Not a signal: a + client-side `"type": "function"` tool merely named `web_search`. + - `server_side_tools` — positive: a tools entry of `"type": "mcp"` (`server_url` / + `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, + `image_generation` used as a tool type, or an Anthropic `code_execution_*` / `text_editor_*` + / `computer_*` / `bash_*` tool type — any provider-executed tool other than web search. Not a + signal: `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, + `tool_choice` — client-side function tools never count. + - `projects` — positive: an `OpenAI-Project` header + (`default_headers={"OpenAI-Project": ...}`), a `project=` kwarg on the OpenAI client, or an + `OPENAI_PROJECT_ID`-style environment name read in code; in usage discovery, a + `project_filter` naming more than one project or a non-default project. Not a signal: + `OpenAI-Organization` alone; no header means the default project. + - `workspaces` — positive: an `anthropic-workspace-id` header or an `ANTHROPIC_WORKSPACE_ID`-style + environment name; in usage discovery, `workspace_ids` naming more than one workspace or a + non-default workspace. Not a signal: OpenRouter's own `workspace_id` scoping, which is an + OpenRouter account concept. +- **Derived by Design, never asked.** `mantle_only_model_or_api` is derived from the model id and + the required API against the access-path tables in `ai-openai-to-bedrock.md` / + `openai-on-bedrock.md`; it is not a scanner output and not a Clarify question. +- **Always asked, never detected.** `in_region_processing` is never code-detectable: nothing in a + call site says whether cross-Region inference is acceptable. Clarify always asks it together + with the Region. + ## Public comparison (read 2026-11-07) | Item | `bedrock-runtime` | `bedrock-mantle` | Source page | From f4037ceaa9a68cb86c88e9ca5004d254814c7872 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 15:04:14 -0800 Subject: [PATCH 32/41] docs(aws-startup-advisor): agent-advisor handoff carries mantle_requirement so gcp Design emits the PR 585 endpoint fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR 585 made exactly one of bedrock_models[].mantle_requirement / endpoint_assumption REQUIRED on every design row (schema-design-aws-ai.md, Design Step 0.8). The agent-advisor handoff translation only copied mantle_selector, so an advisor-driven gcp-to-aws Design had no recorded requirement to cite on Mantle rows and no rule for runtime rows. - handoff-migration.md Step 1: every mantle_* unit carries a mantle_requirement object in the ai_constraints.mantle_requirements.value[] shape ({workload_id: unit_id, selector, source, evidence?, region?}); source is "user" for an explicit requirements.mantle_selector or the in_region_required answer (with region), "detected" with file:line evidence for the OpenAI module's derived web_search selector, "catalog" for mantle_only_model_or_api. Translation table: mantle_* rows copy mantle_requirement verbatim (missing = mismatch); runtime_* rows record the endpoint_assumption sentence with the unit_id as . - gcp-to-aws design-ai.md handoff paragraph: copy mantle_requirement verbatim as the Step 0.8 recorded requirement (no re-derivation from the source API); advisor runtime_* units keep the Step 0.8 rule 4 endpoint_assumption. - migration-plan.md Step 3.5 table: same translation; mantle_requirement is not part of the comparator. No script changes. Verification (this worktree, after the rebase): - skills/agent-advisor/scripts: uv run --group dev pytest -q -> 719 passed - skills/llm-to-bedrock/scripts: uv run --group dev pytest -q -> 655 passed - python3 tools/sync-vendored.py --check -> PASS (65 files) - python3 -m pytest tests/ tools/test_sync_vendored.py -> 1504 passed, 1 failed (tests/test_model_id_lint.py::test_real_plugin_passes flags a retired model id inside skills/llm-to-bedrock/scripts/.venv/.../anthropic/_constants.py — an untracked local venv, unrelated to this change) --- .../references/handoff/handoff-migration.md | 26 +++++++++++++++---- .../phases/migration-plan/migration-plan.md | 7 ++++- .../references/phases/design/design-ai.md | 12 ++++++--- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/handoff/handoff-migration.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/handoff/handoff-migration.md index dea523fc4..8dcd05bad 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/handoff/handoff-migration.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/handoff/handoff-migration.md @@ -17,17 +17,33 @@ signals + eliminations, from design.json's `scores`/`eliminated`). The summary additionally carries, per model-bearing unit, the accepted `api_path`, `invocation_model_id` and `mantle_selector` from `confirm.json` / `design.json` -(`model_decision` / `model_recommendation`), plus this translation table so that `gcp-to-aws` -Design and `llm-to-bedrock` write `migration_path` / `required_client_api` / `mantle_selector` -without re-deciding the endpoint (the vocabulary is -`references/vendored/ai/bedrock-endpoint-policy.md`): +(`model_decision` / `model_recommendation`), and on every `mantle_*` unit a `mantle_requirement` +object in the shape of `ai_constraints.mantle_requirements.value[]` / +`bedrock_models[].mantle_requirement` (`references/vendored/ai/schema-design-aws-ai.md`): +`{ "workload_id": "", "selector": , "source": ..., "evidence"?, "region"? }`. +`source` names how the advisor learned the selector — `"user"` when `model-recommendation-input.json` +carried an explicit `requirements.mantle_selector` (Model Recommend records one only when the +user confirmed the need) or when `in_region_processing` was derived from the user's +`data_residency: in_region_required` answer (then `region` = the run's `region`); `"detected"` +with `evidence` = the `file:line construct` behind the `detected` feature status when the OpenAI +module derived `web_search` from a detected Web Search tool; `"catalog"` with `evidence` = the +catalog file and row for `mantle_only_model_or_api`. A unit on any `runtime_*` path carries no +`mantle_requirement`; downstream Design writes the `endpoint_assumption` sentence for it. + +The summary also carries this translation table so that `gcp-to-aws` Design and +`llm-to-bedrock` write `migration_path` / `required_client_api` / `mantle_selector` / +`mantle_requirement` / `endpoint_assumption` without re-deciding the endpoint (the vocabulary is +`references/vendored/ai/bedrock-endpoint-policy.md`; the advisor contract is the recorded +requirement that Design Step 0.8 would otherwise read from Clarify's +`ai_constraints.mantle_requirements`): - `runtime_openai_responses` → `migration_path: "runtime_openai_cris"`, `required_client_api: "responses"` - `runtime_openai_chat` → `migration_path: "runtime_openai_cris"`, `required_client_api: "chat_completions"` - `runtime_converse` on a proprietary GPT model (`openai.gpt-6*`, `openai.gpt-5.6-*`, `openai.gpt-5.5`, `openai.gpt-5.4`) → `migration_path: "runtime_openai_cris"`, `required_client_api: "converse"` - `runtime_messages` → `migration_path: "runtime_messages"`, `required_client_api: "messages"`, `aws_model_id` = the advisor `invocation_model_id` -- `mantle_openai_responses` / `mantle_openai_chat` / `mantle_messages` → the same `migration_path`, `required_client_api` `responses` / `chat_completions` / `messages`, and `bedrock_models[].mantle_selector` copied verbatim (an advisor Mantle contract without a selector is a mismatch — return to Model Recommend) +- `mantle_openai_responses` / `mantle_openai_chat` / `mantle_messages` → the same `migration_path`, `required_client_api` `responses` / `chat_completions` / `messages`, `bedrock_models[].mantle_selector` and `bedrock_models[].mantle_requirement` copied verbatim (an advisor Mantle contract without a selector or without a `mantle_requirement` is a mismatch — return to Model Recommend) - any other path (`runtime_converse` on a Claude or open-weight model, `runtime_invoke`) → `migration_path` equal to the advisor `api_path`, no `mantle_selector` +- every `runtime_*` unit → the row records `bedrock_models[].endpoint_assumption` verbatim per `schema-design-aws-ai.md`, with the advisor `unit_id` as ``; no `mantle_requirement` ## Step 2 — Check downstream availability diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/migration-plan/migration-plan.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/migration-plan/migration-plan.md index edf810c0d..56a939005 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/migration-plan/migration-plan.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/migration-plan/migration-plan.md @@ -365,7 +365,12 @@ vocabulary of `references/vendored/ai/bedrock-endpoint-policy.md` on the matched | `runtime_openai_chat` | `runtime_openai_cris` | `chat_completions` | same | | `runtime_converse` on a proprietary GPT model (`openai.gpt-6*`, `openai.gpt-5.6-*`, `openai.gpt-5.5`, `openai.gpt-5.4`) | `runtime_openai_cris` | `converse` | same; `runtime_converse` on a Claude or open-weight model keeps the exact comparison | | `runtime_messages` | `runtime_messages` | `messages` | `aws_model_id` = advisor `invocation_model_id` | -| `mantle_openai_responses` / `mantle_openai_chat` / `mantle_messages` | the same path name | `responses` / `chat_completions` / `messages` | `bedrock_models[].mantle_selector` copied verbatim from the advisor contract; an advisor Mantle contract whose `mantle_selector` is null is a mismatch (only a legacy contract written before the field existed keeps the exact model/path comparison) | +| `mantle_openai_responses` / `mantle_openai_chat` / `mantle_messages` | the same path name | `responses` / `chat_completions` / `messages` | `bedrock_models[].mantle_selector` and `bedrock_models[].mantle_requirement` (`handoff-migration.md` Step 1 shape) copied verbatim from the advisor contract; an advisor Mantle contract whose `mantle_selector` is null is a mismatch (only a legacy contract written before the field existed keeps the exact model/path comparison) | + +Every `runtime_*` row carries the `endpoint_assumption` sentence of +`references/vendored/ai/schema-design-aws-ai.md` with the advisor `unit_id` as `` and +no `mantle_requirement`; a `mantle_*` row's `mantle_requirement` is not part of the comparison +below. An absent `runtime_api` on a legacy runtime plan means Converse; an explicit Chat/Responses choice is not equivalent. Do not replace a confirmed Global profile with US Geo or strip an ARN. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index d851f9ca9..ab0a7eeb4 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -508,9 +508,15 @@ Translate every other advisor `api_path` the same way: `runtime_openai_responses `migration_path: "runtime_messages"` + `required_client_api: "messages"` (keep the advisor `invocation_model_id` as `aws_model_id`); `mantle_openai_responses` / `mantle_openai_chat` / `mantle_messages` → the same `migration_path` with `required_client_api` `responses` / -`chat_completions` / `messages` and `bedrock_models[].mantle_selector` copied verbatim from the -advisor contract. An advisor `mantle_*` contract without a `mantle_selector` is a mismatch — -return to Model Recommend rather than inventing one. +`chat_completions` / `messages` and `bedrock_models[].mantle_selector` plus +`bedrock_models[].mantle_requirement` copied verbatim from the advisor contract (the advisor's +`mantle_requirement` — `{ workload_id, selector, source, evidence?, region? }`, `source` ∈ +`user | detected | catalog` — is the recorded requirement Step 0.8 would otherwise read from +`ai_constraints.mantle_requirements`; do not re-derive the endpoint from the source API). An +advisor `mantle_*` contract without a `mantle_selector` or without a `mantle_requirement` is a +mismatch — return to Model Recommend rather than inventing one. Every advisor `runtime_*` unit +keeps the Step 0.8 rule 4 outcome: the row records the `endpoint_assumption` sentence verbatim +with the advisor `unit_id` as ``, and no `mantle_requirement`. **Throughput:** load `references/vendored/ai/ai-migration-guardrails.md` and apply the selected model/endpoint's quota rules. Astra runtime uses input TPM plus 10 times output TPM. From 572fa830fa765c8255954bf2770246ed47687896 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 15:54:02 -0800 Subject: [PATCH 33/41] fix(aws-startup-advisor): agent-advisor OpenAI catalog re-reads the GPT-5.6/5.5/5.4 cards (Mantle Chat, CRIS profile maps, per-API capabilities, Guardrails/server-side-tool flags); fail closed without a callable profile Catalog (skills/shared/ai/models/openai-bedrock.json, vendored copy re-synced): - GPT-5.6 Sol mantle_openai_chat is available; Terra and Luna gain mantle_openai_chat rows (cards read 2026-11-08). The shared path text no longer claims the Sol card lacks Chat Completions. - GPT-5.6 Sol/Terra/Luna carry inference_profiles maps: Sol us./global. only; Terra and Luna us./in./us-gov./global. (no us-gov-* under global). - GPT-5.4 and GPT-5.5 carry per-path capabilities: structured_output_json on runtime_openai_responses only, never on runtime_openai_chat / runtime_converse (cards: "Structured outputs (JSON Schema; Responses API)"). - guardrails_supported on every path row (runtime_converse true; Responses true for GPT-6 Sol/Luna; Chat true for GPT-6 Sol/Luna/6.1 Sol; Mantle false). - server_side_tools_supported=false on GPT-6.1 Sol Mantle rows. - runtime_converse constraint for GPT-5.6 no longer denies structured outputs outright; it records the cards' non-streaming JSON Schema scope. Engine (openai_model_recommendation.py): - When every runtime candidate is skipped because no cataloged CRIS profile is callable from the region under the stated residency, return a decision_required with inference_profile_unverified instead of raising. - _decision_options re-walks a runtime path without the profile requirements so both options are present; the option carries invocation_model_id None and a "resolve a profile" reason suffix. The hard-profile filter now also applies to rows with inference_profile_regions (Astra), so a cross-model fallback never lands on an uncallable profile. - An explicit bare `.` inference_profile_id that names a different model is not echoed on a candidate without a profile matrix. Tests: CodeQL py/incomplete-url-substring-sanitization sites in test_catalog_paths.py and test_openai_model_recommendation.py now use a _cites_aws_docs helper (urlparse scheme == https and hostname == docs.aws.amazon.com). New catalog-shape tests and engine tests for the GPT-5.6 Mantle Chat same-model path, the EU/India/GovCloud profile cases, and the GPT-5.4/5.5 Responses-only structured output scope. Verification: - skills/agent-advisor/scripts: uv run --group dev pytest -q -> 742 passed - python3 tools/sync-vendored.py --check -> PASS (65 files) - tests/test_gpt54_55_cris_prefix_scope.py tests/test_model_catalog_drift.py tests/test_runtime_default_review_round1.py -> 29 passed - plugin tests/: 1495 passed, 1 pre-existing failure (tests/test_model_id_lint.py::test_real_plugin_passes hits the untracked skills/llm-to-bedrock/scripts/.venv; unrelated) - git diff --check clean --- .../vendored/ai/models/openai-bedrock.json | 150 ++++++++---- .../scripts/openai_model_recommendation.py | 101 +++++++- .../scripts/test_catalog_paths.py | 90 ++++++- .../test_openai_model_recommendation.py | 226 ++++++++++++++++-- .../shared/ai/models/openai-bedrock.json | 150 ++++++++---- 5 files changed, 608 insertions(+), 109 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json index ba77539d3..646478d7a 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json @@ -3,7 +3,7 @@ "provider": "openai", "verified_at": "2026-09-24", "verified_region": "us-east-2", - "source": "Opus 5.5 cross-family target: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html (retrieved 2026-09-24). GPT-5.5 and GPT-5.4 rows re-read from their public model cards on 2026-11-06 (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html): both now list bedrock-runtime (Responses, Chat Completions, Converse via us./global. CRIS ids) and Chat Completions on bedrock-mantle. Other OpenAI entries retain their evidence from the superseded 2026-08-21 catalog (removed; see git history). The runtime_openai_responses and runtime_openai_chat path definitions and their per-model rows on every openai_gpt_* entry were added from the nine public model cards and the endpoints / inference-responses-api / inference-chat-completions pages on 2026-11-07; earlier rows and their evidence are unchanged.", + "source": "Opus 5.5 cross-family target: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html (retrieved 2026-09-24). GPT-5.5 and GPT-5.4 rows re-read from their public model cards on 2026-11-06 (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html): both now list bedrock-runtime (Responses, Chat Completions, Converse via us./global. CRIS ids) and Chat Completions on bedrock-mantle. Other OpenAI entries retain their evidence from the superseded 2026-08-21 catalog (removed; see git history). The runtime_openai_responses and runtime_openai_chat path definitions and their per-model rows on every openai_gpt_* entry were added from the nine public model cards and the endpoints / inference-responses-api / inference-chat-completions pages on 2026-11-07; earlier rows and their evidence are unchanged. GPT-5.6 Sol/Terra/Luna Mantle Chat rows, their inference_profiles maps, per-path capabilities for GPT-5.4/5.5, and the guardrails_supported / server_side_tools_supported row flags were re-read from the public model cards on 2026-11-08.", "disclaimer": "The Opus cross-family target is updated to Opus 5.5; the GPT-5.5 and GPT-5.4 entries were re-read from their public model cards on 2026-11-06 (runtime CRIS paths, Chat Completions on mantle, 1.05M context, long-context pricing); the other same-model OpenAI paths and their earlier evidence remain unchanged. This is a migration target catalog, not an account-availability claim. Verify the chosen model, API, region and CRIS profile before implementation. Opus 5.5 uses CRIS on runtime outside eu-west-2, has 1M context/128K output, always-on adaptive thinking, and no Batch tier. Fable and Mythos remain excluded from default selection. Opus 5.5 is the only Opus model in the automatic selection pool; older Opus snapshots and pricing records are historical references. The runtime in-region exception in eu-west-2 was verified 2026-10-03; the original catalog and other model evidence dates remain unchanged.", "paths": { "mantle_openai_responses": { @@ -20,7 +20,7 @@ ], "constraints": [ "Mantle itself has no cross-region form for any GPT model (in-region only); for GPT-5.6, GPT-5.5 and GPT-5.4 the bedrock-runtime CRIS path covers additional regions instead (see runtime_converse)", - "Chat Completions support on Mantle is model-specific: the GPT-5.4 and GPT-5.5 model cards list Chat Completions as supported on bedrock-mantle (read 2026-11-06); the GPT-5.6 Sol card does not. A Chat Completions source reshapes to Responses only when the selected model's card does not list Chat Completions — verify in the target account", + "Chat Completions support on Mantle is model-specific: every GPT-5.6 (Sol, Terra, Luna), GPT-5.5, GPT-5.4 and GPT-6 model card lists Chat Completions as supported on bedrock-mantle (cards read 2026-11-08). A Chat Completions source reshapes to Responses only when the selected model's card leaves Chat Completions unchecked — verify in the target account", "Availability must be probed in the target account and region" ] }, @@ -36,7 +36,7 @@ "OpenAI-compatible Chat Completions surface for models that accept it" ], "constraints": [ - "Model-specific: listed as supported on bedrock-mantle by the GPT-5.4 and GPT-5.5 model cards (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, read 2026-11-06) and by the GPT-6 cards; unsupported on the gpt-5.6-sol model card. Uses /openai/v1/chat/completions", + "Model-specific: listed as supported on bedrock-mantle by the GPT-5.6 Sol, Terra and Luna cards (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html), the GPT-5.4 and GPT-5.5 cards (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html) and the GPT-6 cards (all read 2026-11-08); reshape to Responses only when a model's card leaves Chat Completions unchecked. Uses /openai/v1/chat/completions", "Availability and model support must be probed in the target account and region" ] }, @@ -55,7 +55,7 @@ ], "constraints": [ "GPT-5.5 / GPT-5.4 on runtime are CRIS-only (us. and global. ids; no in-region form) per their model cards (read 2026-11-06); the cards publish In-Region (1.10x) pricing only, so do not assume Global CRIS price parity for them", - "GPT-5.6 on runtime is CRIS-only (no in-region form); Geo CRIS stays within the geography, Global CRIS routes anywhere — pick by data-residency posture. For GPT-5.6, runtime does not support server-side tool use, structured outputs, or application inference profiles; prompt caching is Responses-only", + "GPT-5.6 on runtime is CRIS-only (no in-region form); Geo CRIS stays within the geography, Global CRIS routes anywhere — pick by data-residency posture. For GPT-5.6, runtime does not support server-side tool use; application inference profiles are Converse-only; the GPT-5.6 Sol/Terra/Luna cards (read 2026-11-08) document JSON Schema structured outputs for non-streaming Responses/Chat Completions calls, not yet carried as catalog capabilities; implicit prompt caching applies on Responses/Chat Completions/Invoke/Converse and explicit caching is Responses-only", "GPT-5.5 / GPT-5.4 on runtime (per their model cards, read 2026-11-07): server-side tool use is unsupported; structured outputs and implicit prompt caching are Responses-only; Guardrails and application inference profiles are Converse-only", "Bedrock-native models generally require a Global or geography-scoped CRIS inference profile", "Availability must be probed in the target account and region" @@ -118,13 +118,17 @@ "class": "reasoning", "context_window": 1000000, "output_token_ceiling": "unknown", + "inference_profiles": { + "us": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1"], + "global": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1", "mx-central-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ap-southeast-5", "ap-southeast-6", "ap-southeast-7", "il-central-1", "me-central-1", "me-south-1", "af-south-1", "sa-east-1"] + }, "region_availability": { "commercial": [ "us-east-1", "us-east-2" ], "govcloud": [], - "evidence": "Commercial set per this catalog's pricing cache (US East N. Virginia / Ohio). The 2026-08 GovCloud announcement names Terra and Luna only — Sol's GovCloud availability is not stated; treat as unknown, not absent." + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-mantle in-region rows list us-east-1 and us-east-2; the card lists no GovCloud Mantle row for Sol and the 2026-08 GovCloud announcement names Terra and Luna only — treat Sol's GovCloud availability as unknown, not absent." }, "capabilities": [ "reasoning", @@ -137,31 +141,36 @@ "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": false, + "guardrails_supported": false, "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard" }, "mantle_openai_chat": { - "available": false, + "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": false, - "evidence": "AWS model card: Chat Completions unchecked for this model" + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": true, - "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x" + "guardrails_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): us. from six US/Canada source regions, global. from every listed commercial region; no eu./in./us-gov. profile" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-07): bedrock-runtime lists Responses supported (base URL bedrock-runtime.{region}.amazonaws.com/openai/v1); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol" + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported (base URL bedrock-runtime.{region}.amazonaws.com/openai/v1); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol; the card's runtime table is the source of this model's inference_profiles matrix (us. and global. only); Guardrails Converse-only" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported (/openai/v1/chat/completions); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol" + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported (/openai/v1/chat/completions); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol; the card's runtime table is the source of this model's inference_profiles matrix (us. and global. only); Guardrails Converse-only" } } }, @@ -199,31 +208,39 @@ "available": true, "model_id": "openai.gpt-5.5", "requires_cris": false, + "guardrails_supported": false, "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1 and us-east-2 only; Geo/Global not supported on mantle" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": false, + "guardrails_supported": false, "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": true, - "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.5 (Geo) or global.openai.gpt-5.5 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs" + "guardrails_supported": true, + "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision", "streaming"], + "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.5 (Geo) or global.openai.gpt-5.5 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): 'Response streaming' listed for runtime; 'Structured outputs (JSON Schema; Responses API)' is scoped to Responses, so structured_output_json is not carried on Converse; Guardrails Converse-only" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-07): bedrock-runtime lists Responses supported; structured outputs and implicit prompt caching are Responses-only on runtime; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" + "guardrails_supported": false, + "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision", "structured_output_json", "streaming"], + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): bedrock-runtime lists Responses supported; 'Structured outputs (JSON Schema; Responses API)' and 'Response streaming' listed for runtime, implicit prompt caching Responses-only; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; the card scopes structured outputs and implicit prompt caching to Responses; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" + "guardrails_supported": false, + "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision", "streaming"], + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; 'Response streaming' listed for runtime; the card scopes 'Structured outputs (JSON Schema; Responses API)' and implicit prompt caching to Responses, so structured_output_json is not carried on Chat Completions; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" } } }, @@ -235,6 +252,12 @@ "class": "reasoning", "context_window": 1000000, "output_token_ceiling": "unknown", + "inference_profiles": { + "us": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1"], + "in": ["ap-south-1", "ap-south-2"], + "us-gov": ["us-gov-east-1", "us-gov-west-1"], + "global": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1", "mx-central-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ap-southeast-5", "ap-southeast-6", "ap-southeast-7", "il-central-1", "me-central-1", "me-south-1", "af-south-1", "sa-east-1"] + }, "region_availability": { "commercial": [ "us-east-1", @@ -245,7 +268,7 @@ "us-gov-west-1", "us-gov-east-1" ], - "evidence": "AWS What's New 2026-08 (openai-gpt-terra-luna-govcloud): \"GPT-5.6 Terra and Luna are now generally available on Amazon Bedrock in AWS GovCloud (US-West) and AWS GovCloud (US-East)\"; commercial set per this catalog's pricing cache — verify against the pricing page before relying on it." + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-mantle in-region rows list us-east-1, us-east-2, us-west-2, us-gov-west-1 and us-gov-east-1; consistent with the AWS What's New 2026-08 GovCloud announcement for Terra and Luna." }, "capabilities": [ "reasoning", @@ -257,25 +280,36 @@ "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": false, + "guardrails_supported": false, "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard" }, + "mantle_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.6-terra", + "requires_cris": false, + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2, us-west-2, us-gov-west-1, us-gov-east-1" + }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": true, - "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x" + "guardrails_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): us. from six US/Canada source regions, in. from ap-south-1/ap-south-2, us-gov. from us-gov-east-1/us-gov-west-1 (Global not supported there), global. from every listed commercial region" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-07): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra" + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra" + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" } } }, @@ -287,6 +321,12 @@ "class": "reasoning", "context_window": 1000000, "output_token_ceiling": "unknown", + "inference_profiles": { + "us": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1"], + "in": ["ap-south-1", "ap-south-2"], + "us-gov": ["us-gov-east-1", "us-gov-west-1"], + "global": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1", "mx-central-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ap-southeast-5", "ap-southeast-6", "ap-southeast-7", "il-central-1", "me-central-1", "me-south-1", "af-south-1", "sa-east-1"] + }, "region_availability": { "commercial": [ "us-east-1", @@ -297,7 +337,7 @@ "us-gov-west-1", "us-gov-east-1" ], - "evidence": "AWS What's New 2026-08 (openai-gpt-terra-luna-govcloud): \"GPT-5.6 Terra and Luna are now generally available on Amazon Bedrock in AWS GovCloud (US-West) and AWS GovCloud (US-East)\"; commercial set per this catalog's pricing cache — verify against the pricing page before relying on it." + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-mantle in-region rows list us-east-1, us-east-2, us-west-2, us-gov-west-1 and us-gov-east-1; consistent with the AWS What's New 2026-08 GovCloud announcement for Terra and Luna." }, "capabilities": [ "reasoning", @@ -309,25 +349,36 @@ "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": false, + "guardrails_supported": false, "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard" }, + "mantle_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.6-luna", + "requires_cris": false, + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2, us-west-2, us-gov-west-1, us-gov-east-1" + }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": true, - "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x" + "guardrails_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): us. from six US/Canada source regions, in. from ap-south-1/ap-south-2, us-gov. from us-gov-east-1/us-gov-west-1 (Global not supported there), global. from every listed commercial region" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-07): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna" + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna" + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" } } }, @@ -368,31 +419,40 @@ "available": true, "model_id": "openai.gpt-5.4", "requires_cris": false, + "guardrails_supported": false, "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1, us-east-2, us-west-2 and us-gov-west-1; Geo/Global not supported on mantle" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": false, - "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code" + "guardrails_supported": false, + "capabilities": ["tool_or_function_calling", "reasoning", "streaming", "image_input_vision"], + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): structured outputs are evidenced on the Responses API only, so structured_output_json is not carried on Mantle Chat Completions" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": true, - "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.4 (Geo) or global.openai.gpt-5.4 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs" + "guardrails_supported": true, + "capabilities": ["tool_or_function_calling", "reasoning", "streaming", "image_input_vision"], + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.4 (Geo) or global.openai.gpt-5.4 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): 'Response streaming' listed for runtime; 'Structured outputs (JSON Schema; Responses API)' is scoped to Responses, so structured_output_json is not carried on Converse; Guardrails Converse-only" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-07): bedrock-runtime lists Responses supported; structured outputs and implicit prompt caching are Responses-only on runtime; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" + "guardrails_supported": false, + "capabilities": ["tool_or_function_calling", "structured_output_json", "reasoning", "streaming", "image_input_vision"], + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): bedrock-runtime lists Responses supported; 'Structured outputs (JSON Schema; Responses API)' and 'Response streaming' listed for runtime, implicit prompt caching Responses-only; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; the card scopes structured outputs and implicit prompt caching to Responses; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" + "guardrails_supported": false, + "capabilities": ["tool_or_function_calling", "reasoning", "streaming", "image_input_vision"], + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; 'Response streaming' listed for runtime; the card scopes 'Structured outputs (JSON Schema; Responses API)' and implicit prompt caching to Responses, so structured_output_json is not carried on Chat Completions; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" } } }, @@ -450,6 +510,7 @@ "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": false, + "guardrails_supported": false, "supported_regions": [ "us-east-1", "us-west-2" @@ -466,6 +527,7 @@ "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": false, + "guardrails_supported": false, "supported_regions": [ "us-east-1", "us-west-2" @@ -480,6 +542,7 @@ "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": true, + "guardrails_supported": true, "supported_regions": [ "us-east-1", "us-east-2", @@ -539,6 +602,7 @@ "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": true, + "guardrails_supported": false, "supported_regions": [ "us-east-1", "us-east-2", @@ -599,6 +663,7 @@ "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": true, + "guardrails_supported": false, "supported_regions": [ "us-east-1", "us-east-2", @@ -677,11 +742,11 @@ "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision"], "capability_source": "AWS GPT-6 Sol model card, retrieved 2026-10-10: text and image input, tool calling, 1.05M context and 128K output.", "paths": { - "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, - "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported."}, - "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-07): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"}, - "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"} + "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, + "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, + "runtime_converse": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported. (Guardrails: Responses/Chat Completions/InvokeModel/Converse per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html; card read 2026-11-08)"}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"} } }, "openai_gpt_6_luna": { @@ -704,11 +769,11 @@ "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision"], "capability_source": "AWS GPT-6 Luna model card, retrieved 2026-10-10: text and image input, tool calling, 1.05M context and 128K output.", "paths": { - "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, - "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke."}, - "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-07): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"}, - "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"} + "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, + "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, + "runtime_converse": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke. (Guardrails: Responses/Chat Completions/InvokeModel/Converse per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html; card read 2026-11-08)"}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"} } }, "openai_gpt_6_1_sol": { @@ -731,11 +796,11 @@ "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision"], "capability_source": "AWS GPT-6.1 Sol model card, retrieved 2026-10-10: text and image input, tool calling, 1.05M context and 131072 output.", "paths": { - "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, - "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported."}, - "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-07): bedrock-runtime lists Responses supported; Guardrails are listed for Chat Completions/InvokeModel/Converse but not Responses, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"}, - "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; Guardrails listed on Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"} + "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "guardrails_supported": false, "server_side_tools_supported": false, "evidence": "AWS model card: Mantle Responses in us-east-1. https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): Server-side system tools listed as Not Supported on bedrock-mantle"}, + "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "guardrails_supported": false, "server_side_tools_supported": false, "evidence": "AWS model card: Mantle Chat Completions supported. https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): Server-side system tools listed as Not Supported on bedrock-mantle"}, + "runtime_converse": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported. (Guardrails: Chat Completions/InvokeModel/Converse, not Responses, per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html; card read 2026-11-08)"}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": false, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails are listed for Chat Completions/InvokeModel/Converse but not Responses, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails listed on Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"} } }, "anthropic_claude_sonnet_5": { @@ -757,6 +822,7 @@ "available": true, "model_id": "anthropic.claude-sonnet-5", "requires_cris": true, + "guardrails_supported": true, "evidence": "AWS model card (2026-07-21): Converse on bedrock-runtime checked; Geo (us/eu/au) and Global CRIS supported" } } @@ -779,6 +845,7 @@ "available": true, "model_id": "anthropic.claude-haiku-4-5-20251001-v1:0", "requires_cris": true, + "guardrails_supported": true, "evidence": "anthropic-bedrock.json catalog (verified 2026-09-24) runtime_converse entry (requires CRIS profile)" } } @@ -872,7 +939,8 @@ "runtime_converse": { "available": true, "model_id": "anthropic.claude-opus-5-5", - "requires_cris": true + "requires_cris": true, + "guardrails_supported": true } }, "generation": "bedrock_native", diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py index b64bd1fa7..d8bf468d3 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py @@ -307,7 +307,6 @@ def _catalog_model_for_path(catalog, path, detected_features=None, requirements= continue if ( path.startswith("runtime_") - and not path_config.get("inference_profile_regions") and path_config.get("requires_cris") and _hard_profile_requirement(requirements or {}) and _resolve_invocation_model_id( @@ -327,6 +326,38 @@ def _catalog_model_for_path(catalog, path, detected_features=None, requirements= return first, first_unmet +_PROFILE_REQUIREMENT_KEYS = frozenset({"inference_profile_id", "data_residency", "cris_geography"}) + + +def _profile_relaxed_hit(catalog, path, detected, requirements, candidate_order, region): + """When every runtime candidate was skipped because no cataloged CRIS profile is + callable from `region` under the stated residency, re-walk the path without the + profile requirements so a decision can still show the candidate. The caller + leaves its invocation id unresolved (None); a profile id is never fabricated.""" + if not (path.startswith("runtime_") and _hard_profile_requirement(requirements or {})): + return None + relaxed = {k: v for k, v in (requirements or {}).items() if k not in _PROFILE_REQUIREMENT_KEYS} + hit, _ = _catalog_model_for_path( + catalog, path, detected, relaxed, candidate_order=candidate_order, region=region, + ) + return hit + + +def _unresolved_profile_suffix(path, path_config, invocation_model_id, requirements, region): + """Reason suffix for a runtime option whose CRIS profile could not be resolved.""" + if ( + path.startswith("runtime_") + and path_config["requires_cris"] + and invocation_model_id is None + and _hard_profile_requirement(requirements) + ): + return ( + f" No cataloged CRIS profile for this model is callable from {region} under " + "the stated residency; resolve a profile before this option is implementable." + ) + return "" + + def _resolve_invocation_model_id(model_id, requires_cris, requirements, model=None, region=None): if (model or {}).get("inference_profiles"): return anthropic_model_recommendation._resolve_invocation_model_id( @@ -336,7 +367,12 @@ def _resolve_invocation_model_id(model_id, requires_cris, requirements, model=No return model_id explicit = requirements.get("inference_profile_id") if explicit: - return explicit + # Without a cataloged profile matrix the explicit id cannot be verified, but a + # bare `.` form that names a different model is never this + # candidate's invocation id. ARNs are kept for the verifier to check. + if explicit.startswith("arn:") or explicit.endswith(f".{model_id}"): + return explicit + return None residency = requirements.get("data_residency", "unknown") if residency == "global_allowed": return f"global.{model_id}" @@ -977,6 +1013,10 @@ def _decision_options(catalog, workload, region, selector=None): region="unknown", ) requires_region_change = openai_hit is not None + if openai_hit is None and openai_path.startswith("runtime_openai_"): + openai_hit = _profile_relaxed_hit( + catalog, openai_path, detected, requirements, candidate_order, region + ) if openai_hit: model_key, model, path_config = openai_hit api_label = ( @@ -1010,6 +1050,9 @@ def _decision_options(catalog, workload, region, selector=None): invocation_model_id = _resolve_invocation_model_id( path_config["model_id"], path_config["requires_cris"], requirements, model, region, ) + reason += _unresolved_profile_suffix( + openai_path, path_config, invocation_model_id, requirements, region + ) options.append( { "model_key": model_key, @@ -1023,17 +1066,34 @@ def _decision_options(catalog, workload, region, selector=None): "mantle_selector": selector if openai_path.startswith("mantle_") else None, } ) + converse_order = _converse_candidate_order(workload["source"], workload["requirements"]) runtime, _ = _catalog_model_for_path( catalog, "runtime_converse", detected, workload["requirements"], - candidate_order=_converse_candidate_order(workload["source"], workload["requirements"]), - region=region, + candidate_order=converse_order, region=region, ) + if runtime is None: + runtime = _profile_relaxed_hit( + catalog, "runtime_converse", detected, workload["requirements"], converse_order, + region, + ) if runtime: model_key, model, path_config = runtime invocation_model_id = _resolve_invocation_model_id( path_config["model_id"], path_config["requires_cris"], workload["requirements"], model, region, ) + reason = ( + "SAME-MODEL governance path: this OpenAI target runs on bedrock-runtime via a " + "CRIS id — Guardrails (Converse API only) and invocation logging without a " + "model change. Check the model's region/profile matrix and dated prices." + if model_key == _same_model_runtime_key(workload["source"]) + else "Uses Bedrock-native Converse request/response shapes and a " + "cross-family model; requires rewriting the OpenAI integration." + ) + reason += _unresolved_profile_suffix( + "runtime_converse", path_config, invocation_model_id, workload["requirements"], + region, + ) options.append( { "model_key": model_key, @@ -1043,14 +1103,7 @@ def _decision_options(catalog, workload, region, selector=None): "requires_cris": anthropic_model_recommendation._invocation_requires_cris( path_config, invocation_model_id ), - "reason": ( - "SAME-MODEL governance path: this OpenAI target runs on bedrock-runtime via a " - "CRIS id — Guardrails (Converse API only) and invocation logging without a " - "model change. Check the model's region/profile matrix and dated prices." - if model_key == _same_model_runtime_key(workload["source"]) - else "Uses Bedrock-native Converse request/response shapes and a " - "cross-family model; requires rewriting the OpenAI integration." - ), + "reason": reason, "mantle_selector": None, } ) @@ -1298,6 +1351,30 @@ def _decision_conflict(reason_head, reasons, message, remediation, options): ), path, ) + if _hard_profile_requirement(requirements): + # Every runtime candidate was skipped because its cataloged CRIS + # profile matrix has no profile callable from this region under the + # stated residency (e.g. an EU-Geo caller on models that publish + # us./global. only). Fail closed with a decision rather than raising. + return _unresolved( + rationale_head, + [ + f"No {path} candidate publishes a CRIS profile callable from {region} " + "under the stated residency (data_residency / cris_geography / " + "inference_profile_id)." + ], + _finding( + "inference_profile_unverified", + "[BLOCKS]", + f"No {path} candidate has a cataloged CRIS profile for calls from " + f"{region} under the stated residency.", + "Choose a Geo or Global profile the selected model publishes for this " + "source region, change the region or geography, supply a verified " + "explicit inference_profile_id, or take the runtime Converse option, " + "then rerun Model Recommend.", + ), + path, + ) raise ValueError(f"catalog has no available model for path {path}") model_key, model, path_config = catalog_hit diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_catalog_paths.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_catalog_paths.py index d5855deb0..90869d231 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_catalog_paths.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_catalog_paths.py @@ -9,6 +9,7 @@ import json import pathlib import re +import urllib.parse import pytest @@ -17,12 +18,26 @@ SCHEMAS = pathlib.Path(model_recommendation.__file__).parent / "schemas" RUNTIME_SDK_PATHS = ("runtime_openai_responses", "runtime_openai_chat", "runtime_messages") +GPT_56_MODELS = ("openai_gpt_5_6_sol", "openai_gpt_5_6_terra", "openai_gpt_5_6_luna") +_URL = re.compile(r"https?://[^\s)\]]+") +_READ_2026_11_07_OR_08 = re.compile(r"read 2026-11-0[78]") def _schema(name): return json.loads((SCHEMAS / name).read_text()) +def _cites_aws_docs(text): + """True when `text` contains an https URL whose host is exactly + docs.aws.amazon.com (hostname equality after urlparse, not a substring).""" + urls = _URL.findall(text or "") + return any( + urllib.parse.urlparse(url).scheme == "https" + and urllib.parse.urlparse(url).hostname == "docs.aws.amazon.com" + for url in urls + ) + + @pytest.fixture(scope="module") def catalogs(): return { @@ -124,5 +139,78 @@ def test_new_runtime_rows_cite_public_docs_with_a_read_date(catalogs): if path not in RUNTIME_SDK_PATHS: continue evidence = row.get("evidence", "") - assert "docs.aws.amazon.com" in evidence, f"{name}: {model_key}.{path}" + assert _cites_aws_docs(evidence), f"{name}: {model_key}.{path}" assert read_date.search(evidence), f"{name}: {model_key}.{path} lacks a read date" + + +# --- GPT-5.6 Sol/Terra/Luna re-read from their public model cards (2026-11-08) --- + +@pytest.mark.parametrize("model_key", GPT_56_MODELS) +def test_gpt56_models_list_mantle_chat_with_dated_card_evidence(catalogs, model_key): + row = catalogs["openai"]["models"][model_key]["paths"]["mantle_openai_chat"] + assert row["available"] is True, model_key + assert row["requires_cris"] is False, model_key + assert _cites_aws_docs(row["evidence"]), model_key + assert _READ_2026_11_07_OR_08.search(row["evidence"]), model_key + + +def test_gpt56_inference_profile_matrices_follow_the_cards(catalogs): + models = catalogs["openai"]["models"] + us_sources = models["openai_gpt_5_4"]["inference_profiles"]["us"] + sol = models["openai_gpt_5_6_sol"]["inference_profiles"] + assert set(sol) == {"us", "global"} + assert sol["us"] == us_sources + for model_key in ("openai_gpt_5_6_terra", "openai_gpt_5_6_luna"): + profiles = models[model_key]["inference_profiles"] + assert set(profiles) == {"us", "in", "us-gov", "global"}, model_key + assert profiles["us"] == us_sources, model_key + assert profiles["in"] == ["ap-south-1", "ap-south-2"], model_key + assert profiles["us-gov"] == ["us-gov-east-1", "us-gov-west-1"], model_key + assert not any(region.startswith("us-gov-") for region in profiles["global"]), model_key + + +def test_every_gpt_path_row_carries_a_boolean_guardrails_flag(catalogs): + models = catalogs["openai"]["models"] + responses_true = set() + chat_true = set() + for model_key, model in models.items(): + if model.get("generation") == "bedrock_native": + continue + for path, row in model["paths"].items(): + assert isinstance(row.get("guardrails_supported"), bool), f"{model_key}.{path}" + if path.startswith("mantle_"): + assert row["guardrails_supported"] is False, f"{model_key}.{path}" + if path == "runtime_converse": + assert row["guardrails_supported"] is True, f"{model_key}.{path}" + if path == "runtime_openai_responses" and row["guardrails_supported"]: + responses_true.add(model_key) + if path == "runtime_openai_chat" and row["guardrails_supported"]: + chat_true.add(model_key) + assert responses_true == {"openai_gpt_6_sol", "openai_gpt_6_luna"} + assert chat_true == {"openai_gpt_6_sol", "openai_gpt_6_luna", "openai_gpt_6_1_sol"} + + +def test_only_gpt61_sol_mantle_rows_deny_server_side_tools(catalogs): + models = catalogs["openai"]["models"] + denied = { + (model_key, path) + for model_key, model in models.items() + for path, row in model["paths"].items() + if row.get("server_side_tools_supported") is False + } + assert denied == { + ("openai_gpt_6_1_sol", "mantle_openai_responses"), + ("openai_gpt_6_1_sol", "mantle_openai_chat"), + } + for path in ("mantle_openai_responses", "mantle_openai_chat"): + row = models["openai_gpt_6_1_sol"]["paths"][path] + assert _cites_aws_docs(row["evidence"]) + assert "read 2026-11-08" in row["evidence"] + + +@pytest.mark.parametrize("model_key", ("openai_gpt_5_4", "openai_gpt_5_5")) +def test_gpt54_gpt55_scope_structured_outputs_to_the_responses_api(catalogs, model_key): + paths = catalogs["openai"]["models"][model_key]["paths"] + assert "structured_output_json" in paths["runtime_openai_responses"]["capabilities"] + assert "structured_output_json" not in paths["runtime_openai_chat"]["capabilities"] + assert "structured_output_json" not in paths["runtime_converse"]["capabilities"] diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py index b1f4810b8..6a565409f 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py @@ -9,6 +9,8 @@ import json import pathlib +import re +import urllib.parse import jsonschema import pytest @@ -19,6 +21,18 @@ SCRIPTS = pathlib.Path(model_recommendation.__file__).parent OPENAI_CATALOG = model_recommendation.load_openai_catalog() +_URL = re.compile(r"https?://[^\s)\]]+") + + +def _cites_aws_docs(text): + """True when `text` contains an https URL whose host is exactly + docs.aws.amazon.com (hostname equality after urlparse, not a substring).""" + urls = _URL.findall(text or "") + return any( + urllib.parse.urlparse(url).scheme == "https" + and urllib.parse.urlparse(url).hostname == "docs.aws.amazon.com" + for url in urls + ) def _workload(**overrides): @@ -199,6 +213,11 @@ def test_runtime_converse_path_text_does_not_grant_in_profile_to_gpt54_gpt55(): assert "GPT-5.6 Terra and Luna publish us./in./global. ids" in strengths assert "us./global. only for GPT-5.6 Sol, GPT-5.5 and GPT-5.4" in strengths assert "For GPT-5.6, runtime does not support server-side tool use" in constraints + # The GPT-5.6 cards (read 2026-11-08) document non-streaming JSON Schema on + # Responses/Chat Completions; the sentence must not deny structured outputs + # outright, nor claim them as catalog capabilities. + assert "does not support server-side tool use, structured outputs" not in constraints + assert "not yet carried as catalog capabilities" in constraints assert "GPT-5.5 / GPT-5.4 on runtime" in constraints assert "structured outputs and implicit prompt caching are Responses-only" in constraints assert "Guardrails and application inference profiles are Converse-only" in constraints @@ -207,8 +226,8 @@ def test_runtime_converse_path_text_does_not_grant_in_profile_to_gpt54_gpt55(): @pytest.mark.parametrize("source_id,requirements,path", [ # A Chat source explicitly moved to bedrock-runtime Responses. ("gpt-5.4", {"preferred_api_path": "runtime_openai_responses"}, "runtime_openai_responses"), - # A Chat source on Mantle Responses by selector + preference (GPT-5.6 Sol's - # Mantle Chat row is unavailable, so Responses is its only Mantle API). + # A Chat source explicitly moved to Mantle Responses by selector + preference + # (GPT-5.6 Sol's card lists Mantle Chat too, so this reshape is a choice). ("gpt-5.6-sol", {"preferred_api_path": "mantle_openai_responses", "mantle_selector": "background_inference"}, "mantle_openai_responses"), ]) @@ -231,22 +250,42 @@ def test_chat_completions_case_includes_full_reshape_deltas(source_id, requireme assert "chat_to_responses_state" in codes -def test_chat_source_with_selector_keeps_chat_on_an_available_mantle_chat_model(): - # GPT-5.6 Sol has no Mantle Chat row; the walk keeps the Chat surface on the - # next same-family model whose Mantle Chat row is in region, not a reshape. +@pytest.mark.parametrize("source_id", ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"]) +def test_gpt56_chat_source_with_selector_keeps_its_own_model_on_mantle_chat(source_id): + # The GPT-5.6 Sol, Terra and Luna cards (read 2026-11-08) list Chat Completions + # on bedrock-mantle, so a Chat source with a selector stays on the same model + # and the same API — no fallback, no reshape. rec = _recommend( _workload( - source={"model_ids": ["gpt-5.6-sol"], "api_surface": "chat_completions"}, - requirements={"mantle_selector": "background_inference"}, + source={"model_ids": [source_id], "api_surface": "chat_completions"}, + requirements={"mantle_selector": "projects"}, ), region="us-east-1", ) + assert rec["decision_status"] == "recommended" assert rec["api_path"] == "mantle_openai_chat" - assert rec["primary_model"] == "openai.gpt-5.5" - assert "fallback" in rec["rationale"][0] + assert rec["primary_model"] == f"openai.{source_id}" + assert rec["mantle_selector"] == "projects" + assert "fallback" not in rec["rationale"][0] assert "chat_completions_to_responses_required" not in _codes(rec["blocks"]) +def test_gpt56_sol_chat_with_selector_outside_its_mantle_footprint_falls_to_terra(): + # The region gate is unchanged: Sol's Mantle footprint is us-east-1/us-east-2, + # so a GovCloud caller lands on Terra's in-region Mantle Chat row. + rec = _recommend( + _workload( + source={"model_ids": ["gpt-5.6-sol"], "api_surface": "chat_completions"}, + requirements={"mantle_selector": "projects"}, + ), + region="us-gov-west-1", + ) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "mantle_openai_chat" + assert rec["primary_model"] == "openai.gpt-5.6-terra" + assert "fallback" in rec["rationale"][0] + + def test_governance_selects_runtime_converse(): # Cross-family Converse tier: an unrecognized source id (gpt-4) with Guardrails. rec = _recommend( @@ -321,7 +360,7 @@ def test_detected_web_search_derives_the_mantle_selector(): impacts = {a["feature"]: a["impact"] for a in rec["architecture_impacts"]} assert "bedrock-mantle" in impacts["web_search"] assert "passthrough" not in impacts["web_search"].lower() - assert "docs.aws.amazon.com" in impacts["web_search"] + assert _cites_aws_docs(impacts["web_search"]) assert "web_search" not in rec["compatibility"]["rearchitecture"] assert "web_search" in rec["compatibility"]["portable"] @@ -579,7 +618,8 @@ def test_gpt6_chat_with_selector_in_us_west_2_falls_back_to_same_family_astra(so @pytest.mark.parametrize("source_id", ["gpt-6-sol", "gpt-6.1-sol"]) def test_gpt6_chat_with_selector_in_us_east_2_prefers_newest_gpt5_over_gpt54(source_id): # No GPT-6 model lists Mantle Chat in us-east-2 (Astra: us-east-1/us-west-2), - # so the walk falls to the GPT-5 family, newest first: 5.5, never 5.4. + # so the walk falls to the GPT-5 family, newest first: GPT-5.6 Sol (its card + # lists Mantle Chat in us-east-2, read 2026-11-08), never 5.4. rec = _recommend( _workload( source={"model_ids": [source_id], "api_surface": "chat_completions"}, @@ -590,7 +630,7 @@ def test_gpt6_chat_with_selector_in_us_east_2_prefers_newest_gpt5_over_gpt54(sou assert rec["decision_status"] == "recommended" assert rec["api_path"] == "mantle_openai_chat" assert rec["primary_model"] != "openai.gpt-5.4" - assert rec["primary_model"] == "openai.gpt-5.5" + assert rec["primary_model"] == "openai.gpt-5.6-sol" assert "us-east-2" not in OPENAI_CATALOG["models"]["openai_gpt_6_astra"]["region_availability"]["commercial"] assert "fallback" in rec["rationale"][0] @@ -1101,13 +1141,14 @@ def test_g02_native_only_lists_catalog_evidenced_features(): def test_g02_evidence_driven_fallback_skips_unevidenced_newer_models(): # A gpt-5.1 source has no same-model identity; the walk starts at GPT-5.6 Sol - # (no structured-output evidence) and falls to GPT-5.4, which has it. + # (no structured-output evidence) and falls to GPT-5.5, whose runtime Responses + # row carries it (card read 2026-11-08), ahead of GPT-5.4. rec = _recommend( _workload(source={"model_ids": ["gpt-5.1"]}, detected_features=["structured_output_json"]) ) assert rec["decision_status"] == "recommended" - assert rec["primary_model"] == "openai.gpt-5.4" + assert rec["primary_model"] == "openai.gpt-5.5" assert "structured_output_json" in rec["compatibility"]["native"] @@ -1703,3 +1744,160 @@ def test_astra_rejects_capacity_above_documented_limit(requirement, limit): ) assert rec["decision_status"] == "decision_required" assert "unverified_capacity" in _codes(rec["blocks"]) + + +# --- CRIS profile matrices for GPT-5.6 Sol/Terra/Luna (cards read 2026-11-08) -- +# +# Sol publishes us./global. only; Terra and Luna publish us./in./us-gov./global. +# A runtime path must never echo a profile id the card does not publish, and when +# no candidate publishes a callable profile the engine fails closed with a decision. + +def _invocation_ids(rec): + ids = {rec.get("invocation_model_id")} + ids |= {option["invocation_model_id"] for option in rec.get("decision_options") or []} + return ids + + +def test_eu_geo_caller_on_us_only_models_fails_closed_without_fabricated_eu_profile(): + workload = _workload(requirements={"data_residency": "geo_required", "cris_geography": "eu"}) + out = _full_recommend(workload, region="eu-west-1") + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == "decision_required" + assert "inference_profile_unverified" in _codes(rec["blocks"]) + assert not any( + (invocation_id or "").startswith("eu.openai.") for invocation_id in _invocation_ids(rec) + ) + options = {option["api_path"]: option for option in rec["decision_options"]} + assert options["runtime_openai_responses"]["invocation_model_id"] is None + assert "resolve a profile" in options["runtime_openai_responses"]["reason"] + + +def test_explicit_eu_profile_for_gpt56_sol_is_never_echoed(): + rec = _recommend( + _workload( + source={"model_ids": ["gpt-5.6-sol"]}, + requirements={"inference_profile_id": "eu.openai.gpt-5.6-sol", + "api_continuity": "required"}, + ), + region="eu-west-1", + ) + assert rec["invocation_model_id"] is None + if rec["decision_status"] == "recommended": + assert _codes(rec["blocks"]) & {"cris_residency_unresolved", "inference_profile_unverified"} + assert "eu.openai.gpt-5.6-sol" not in _invocation_ids(rec) + + +def test_gpt56_terra_accepts_the_india_geo_profile_from_ap_south_1(): + rec = _recommend( + _workload( + source={"model_ids": ["gpt-5.6-terra"]}, + requirements={"data_residency": "geo_required", + "inference_profile_id": "in.openai.gpt-5.6-terra"}, + ), + region="ap-south-1", + ) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "runtime_openai_responses" + assert rec["primary_model"] == "openai.gpt-5.6-terra" + assert rec["invocation_model_id"] == "in.openai.gpt-5.6-terra" + assert "cris_residency_unresolved" not in _codes(rec["blocks"]) + + +def test_gpt56_luna_accepts_the_govcloud_profile_from_us_gov_west_1(): + rec = _recommend( + _workload( + source={"model_ids": ["gpt-5.6-luna"]}, + requirements={"inference_profile_id": "us-gov.openai.gpt-5.6-luna"}, + ), + region="us-gov-west-1", + ) + assert rec["decision_status"] == "recommended" + assert rec["invocation_model_id"] == "us-gov.openai.gpt-5.6-luna" + + +def test_gpt56_luna_in_govcloud_has_no_global_profile(): + # The Luna card lists no Global profile for GovCloud callers (read 2026-11-08). + rec = _recommend( + _workload( + source={"model_ids": ["gpt-5.6-luna"]}, + requirements={"data_residency": "global_allowed"}, + ), + region="us-gov-west-1", + ) + assert rec["invocation_model_id"] is None + assert _codes(rec["blocks"]) & {"cris_residency_unresolved", "inference_profile_unverified"} + assert "global.openai.gpt-5.6-luna" not in _invocation_ids(rec) + + +def test_gpt56_sol_global_profile_from_us_east_1_lists_both_us_and_global(): + rec = _recommend( + _workload( + source={"model_ids": ["gpt-5.6-sol"]}, + requirements={"data_residency": "global_allowed"}, + ), + region="us-east-1", + ) + assert rec["decision_status"] == "recommended" + assert rec["invocation_model_id"] == "global.openai.gpt-5.6-sol" + allowed = set(rec["verification"]["allowed_inference_profiles"]) + assert {"us.openai.gpt-5.6-sol", "global.openai.gpt-5.6-sol"} <= allowed + + +# --- GPT-5.4 / GPT-5.5 per-API capabilities (cards read 2026-11-08) ------------ +# +# Both cards list 'Structured outputs (JSON Schema; Responses API)': the feature is +# evidenced on runtime Responses only, never on Chat Completions or Converse. + +def test_gpt55_structured_output_stays_on_gpt55_runtime_responses(): + rec = _recommend( + _workload( + source={"model_ids": ["gpt-5.5"]}, + requirements={"critical_features": ["structured_output_json"], + "data_residency": "global_allowed"}, + ) + ) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "runtime_openai_responses" + assert rec["primary_model"] == "openai.gpt-5.5" + assert "structured_output_json" in rec["compatibility"]["native"] + + +@pytest.mark.parametrize("path,unmet", [ + ("runtime_openai_chat", ["structured_output_json"]), + ("runtime_converse", ["structured_output_json"]), + ("runtime_openai_responses", []), +]) +def test_gpt54_structured_output_is_evidenced_only_on_runtime_responses(path, unmet): + _hit, missing = oai._catalog_model_for_path( + OPENAI_CATALOG, path, ["structured_output_json"], {}, + candidate_order=["openai_gpt_5_4"], + ) + assert missing == unmet + + +def test_gpt54_guardrails_plus_structured_output_has_no_evidenced_converse_candidate(): + rec = _recommend( + _workload( + requirements={"governance": ["guardrails"], + "critical_features": ["structured_output_json"], + "data_residency": "global_allowed"}, + ) + ) + assert rec["decision_status"] == "decision_required" + assert "unverified_capability" in _codes(rec["blocks"]) + assert "structured_output_json" not in rec["compatibility"]["native"] + + +def test_gpt54_chat_source_with_structured_output_does_not_claim_it_on_gpt54_chat(): + rec = _recommend( + _workload( + source={"api_surface": "chat_completions"}, + requirements={"critical_features": ["structured_output_json"]}, + ), + region="us-east-2", + ) + assert rec["primary_model"] != "openai.gpt-5.4" + if rec["decision_status"] == "recommended": + row = OPENAI_CATALOG["models"][rec["model_identity"]["model_key"]]["paths"]["runtime_openai_chat"] + assert "structured_output_json" in row["capabilities"] diff --git a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json index ba77539d3..646478d7a 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json @@ -3,7 +3,7 @@ "provider": "openai", "verified_at": "2026-09-24", "verified_region": "us-east-2", - "source": "Opus 5.5 cross-family target: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html (retrieved 2026-09-24). GPT-5.5 and GPT-5.4 rows re-read from their public model cards on 2026-11-06 (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html): both now list bedrock-runtime (Responses, Chat Completions, Converse via us./global. CRIS ids) and Chat Completions on bedrock-mantle. Other OpenAI entries retain their evidence from the superseded 2026-08-21 catalog (removed; see git history). The runtime_openai_responses and runtime_openai_chat path definitions and their per-model rows on every openai_gpt_* entry were added from the nine public model cards and the endpoints / inference-responses-api / inference-chat-completions pages on 2026-11-07; earlier rows and their evidence are unchanged.", + "source": "Opus 5.5 cross-family target: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-5-5.html (retrieved 2026-09-24). GPT-5.5 and GPT-5.4 rows re-read from their public model cards on 2026-11-06 (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html): both now list bedrock-runtime (Responses, Chat Completions, Converse via us./global. CRIS ids) and Chat Completions on bedrock-mantle. Other OpenAI entries retain their evidence from the superseded 2026-08-21 catalog (removed; see git history). The runtime_openai_responses and runtime_openai_chat path definitions and their per-model rows on every openai_gpt_* entry were added from the nine public model cards and the endpoints / inference-responses-api / inference-chat-completions pages on 2026-11-07; earlier rows and their evidence are unchanged. GPT-5.6 Sol/Terra/Luna Mantle Chat rows, their inference_profiles maps, per-path capabilities for GPT-5.4/5.5, and the guardrails_supported / server_side_tools_supported row flags were re-read from the public model cards on 2026-11-08.", "disclaimer": "The Opus cross-family target is updated to Opus 5.5; the GPT-5.5 and GPT-5.4 entries were re-read from their public model cards on 2026-11-06 (runtime CRIS paths, Chat Completions on mantle, 1.05M context, long-context pricing); the other same-model OpenAI paths and their earlier evidence remain unchanged. This is a migration target catalog, not an account-availability claim. Verify the chosen model, API, region and CRIS profile before implementation. Opus 5.5 uses CRIS on runtime outside eu-west-2, has 1M context/128K output, always-on adaptive thinking, and no Batch tier. Fable and Mythos remain excluded from default selection. Opus 5.5 is the only Opus model in the automatic selection pool; older Opus snapshots and pricing records are historical references. The runtime in-region exception in eu-west-2 was verified 2026-10-03; the original catalog and other model evidence dates remain unchanged.", "paths": { "mantle_openai_responses": { @@ -20,7 +20,7 @@ ], "constraints": [ "Mantle itself has no cross-region form for any GPT model (in-region only); for GPT-5.6, GPT-5.5 and GPT-5.4 the bedrock-runtime CRIS path covers additional regions instead (see runtime_converse)", - "Chat Completions support on Mantle is model-specific: the GPT-5.4 and GPT-5.5 model cards list Chat Completions as supported on bedrock-mantle (read 2026-11-06); the GPT-5.6 Sol card does not. A Chat Completions source reshapes to Responses only when the selected model's card does not list Chat Completions — verify in the target account", + "Chat Completions support on Mantle is model-specific: every GPT-5.6 (Sol, Terra, Luna), GPT-5.5, GPT-5.4 and GPT-6 model card lists Chat Completions as supported on bedrock-mantle (cards read 2026-11-08). A Chat Completions source reshapes to Responses only when the selected model's card leaves Chat Completions unchecked — verify in the target account", "Availability must be probed in the target account and region" ] }, @@ -36,7 +36,7 @@ "OpenAI-compatible Chat Completions surface for models that accept it" ], "constraints": [ - "Model-specific: listed as supported on bedrock-mantle by the GPT-5.4 and GPT-5.5 model cards (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, read 2026-11-06) and by the GPT-6 cards; unsupported on the gpt-5.6-sol model card. Uses /openai/v1/chat/completions", + "Model-specific: listed as supported on bedrock-mantle by the GPT-5.6 Sol, Terra and Luna cards (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html), the GPT-5.4 and GPT-5.5 cards (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html) and the GPT-6 cards (all read 2026-11-08); reshape to Responses only when a model's card leaves Chat Completions unchecked. Uses /openai/v1/chat/completions", "Availability and model support must be probed in the target account and region" ] }, @@ -55,7 +55,7 @@ ], "constraints": [ "GPT-5.5 / GPT-5.4 on runtime are CRIS-only (us. and global. ids; no in-region form) per their model cards (read 2026-11-06); the cards publish In-Region (1.10x) pricing only, so do not assume Global CRIS price parity for them", - "GPT-5.6 on runtime is CRIS-only (no in-region form); Geo CRIS stays within the geography, Global CRIS routes anywhere — pick by data-residency posture. For GPT-5.6, runtime does not support server-side tool use, structured outputs, or application inference profiles; prompt caching is Responses-only", + "GPT-5.6 on runtime is CRIS-only (no in-region form); Geo CRIS stays within the geography, Global CRIS routes anywhere — pick by data-residency posture. For GPT-5.6, runtime does not support server-side tool use; application inference profiles are Converse-only; the GPT-5.6 Sol/Terra/Luna cards (read 2026-11-08) document JSON Schema structured outputs for non-streaming Responses/Chat Completions calls, not yet carried as catalog capabilities; implicit prompt caching applies on Responses/Chat Completions/Invoke/Converse and explicit caching is Responses-only", "GPT-5.5 / GPT-5.4 on runtime (per their model cards, read 2026-11-07): server-side tool use is unsupported; structured outputs and implicit prompt caching are Responses-only; Guardrails and application inference profiles are Converse-only", "Bedrock-native models generally require a Global or geography-scoped CRIS inference profile", "Availability must be probed in the target account and region" @@ -118,13 +118,17 @@ "class": "reasoning", "context_window": 1000000, "output_token_ceiling": "unknown", + "inference_profiles": { + "us": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1"], + "global": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1", "mx-central-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ap-southeast-5", "ap-southeast-6", "ap-southeast-7", "il-central-1", "me-central-1", "me-south-1", "af-south-1", "sa-east-1"] + }, "region_availability": { "commercial": [ "us-east-1", "us-east-2" ], "govcloud": [], - "evidence": "Commercial set per this catalog's pricing cache (US East N. Virginia / Ohio). The 2026-08 GovCloud announcement names Terra and Luna only — Sol's GovCloud availability is not stated; treat as unknown, not absent." + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-mantle in-region rows list us-east-1 and us-east-2; the card lists no GovCloud Mantle row for Sol and the 2026-08 GovCloud announcement names Terra and Luna only — treat Sol's GovCloud availability as unknown, not absent." }, "capabilities": [ "reasoning", @@ -137,31 +141,36 @@ "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": false, + "guardrails_supported": false, "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard" }, "mantle_openai_chat": { - "available": false, + "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": false, - "evidence": "AWS model card: Chat Completions unchecked for this model" + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": true, - "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x" + "guardrails_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): us. from six US/Canada source regions, global. from every listed commercial region; no eu./in./us-gov. profile" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-07): bedrock-runtime lists Responses supported (base URL bedrock-runtime.{region}.amazonaws.com/openai/v1); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol" + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported (base URL bedrock-runtime.{region}.amazonaws.com/openai/v1); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol; the card's runtime table is the source of this model's inference_profiles matrix (us. and global. only); Guardrails Converse-only" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported (/openai/v1/chat/completions); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol" + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported (/openai/v1/chat/completions); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol; the card's runtime table is the source of this model's inference_profiles matrix (us. and global. only); Guardrails Converse-only" } } }, @@ -199,31 +208,39 @@ "available": true, "model_id": "openai.gpt-5.5", "requires_cris": false, + "guardrails_supported": false, "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1 and us-east-2 only; Geo/Global not supported on mantle" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": false, + "guardrails_supported": false, "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": true, - "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.5 (Geo) or global.openai.gpt-5.5 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs" + "guardrails_supported": true, + "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision", "streaming"], + "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.5 (Geo) or global.openai.gpt-5.5 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): 'Response streaming' listed for runtime; 'Structured outputs (JSON Schema; Responses API)' is scoped to Responses, so structured_output_json is not carried on Converse; Guardrails Converse-only" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-07): bedrock-runtime lists Responses supported; structured outputs and implicit prompt caching are Responses-only on runtime; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" + "guardrails_supported": false, + "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision", "structured_output_json", "streaming"], + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): bedrock-runtime lists Responses supported; 'Structured outputs (JSON Schema; Responses API)' and 'Response streaming' listed for runtime, implicit prompt caching Responses-only; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; the card scopes structured outputs and implicit prompt caching to Responses; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" + "guardrails_supported": false, + "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision", "streaming"], + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; 'Response streaming' listed for runtime; the card scopes 'Structured outputs (JSON Schema; Responses API)' and implicit prompt caching to Responses, so structured_output_json is not carried on Chat Completions; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" } } }, @@ -235,6 +252,12 @@ "class": "reasoning", "context_window": 1000000, "output_token_ceiling": "unknown", + "inference_profiles": { + "us": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1"], + "in": ["ap-south-1", "ap-south-2"], + "us-gov": ["us-gov-east-1", "us-gov-west-1"], + "global": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1", "mx-central-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ap-southeast-5", "ap-southeast-6", "ap-southeast-7", "il-central-1", "me-central-1", "me-south-1", "af-south-1", "sa-east-1"] + }, "region_availability": { "commercial": [ "us-east-1", @@ -245,7 +268,7 @@ "us-gov-west-1", "us-gov-east-1" ], - "evidence": "AWS What's New 2026-08 (openai-gpt-terra-luna-govcloud): \"GPT-5.6 Terra and Luna are now generally available on Amazon Bedrock in AWS GovCloud (US-West) and AWS GovCloud (US-East)\"; commercial set per this catalog's pricing cache — verify against the pricing page before relying on it." + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-mantle in-region rows list us-east-1, us-east-2, us-west-2, us-gov-west-1 and us-gov-east-1; consistent with the AWS What's New 2026-08 GovCloud announcement for Terra and Luna." }, "capabilities": [ "reasoning", @@ -257,25 +280,36 @@ "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": false, + "guardrails_supported": false, "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard" }, + "mantle_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.6-terra", + "requires_cris": false, + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2, us-west-2, us-gov-west-1, us-gov-east-1" + }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": true, - "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x" + "guardrails_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): us. from six US/Canada source regions, in. from ap-south-1/ap-south-2, us-gov. from us-gov-east-1/us-gov-west-1 (Global not supported there), global. from every listed commercial region" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-07): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra" + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra" + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" } } }, @@ -287,6 +321,12 @@ "class": "reasoning", "context_window": 1000000, "output_token_ceiling": "unknown", + "inference_profiles": { + "us": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1"], + "in": ["ap-south-1", "ap-south-2"], + "us-gov": ["us-gov-east-1", "us-gov-west-1"], + "global": ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ca-central-1", "ca-west-1", "mx-central-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ap-southeast-5", "ap-southeast-6", "ap-southeast-7", "il-central-1", "me-central-1", "me-south-1", "af-south-1", "sa-east-1"] + }, "region_availability": { "commercial": [ "us-east-1", @@ -297,7 +337,7 @@ "us-gov-west-1", "us-gov-east-1" ], - "evidence": "AWS What's New 2026-08 (openai-gpt-terra-luna-govcloud): \"GPT-5.6 Terra and Luna are now generally available on Amazon Bedrock in AWS GovCloud (US-West) and AWS GovCloud (US-East)\"; commercial set per this catalog's pricing cache — verify against the pricing page before relying on it." + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-mantle in-region rows list us-east-1, us-east-2, us-west-2, us-gov-west-1 and us-gov-east-1; consistent with the AWS What's New 2026-08 GovCloud announcement for Terra and Luna." }, "capabilities": [ "reasoning", @@ -309,25 +349,36 @@ "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": false, + "guardrails_supported": false, "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard" }, + "mantle_openai_chat": { + "available": true, + "model_id": "openai.gpt-5.6-luna", + "requires_cris": false, + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2, us-west-2, us-gov-west-1, us-gov-east-1" + }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": true, - "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x" + "guardrails_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): us. from six US/Canada source regions, in. from ap-south-1/ap-south-2, us-gov. from us-gov-east-1/us-gov-west-1 (Global not supported there), global. from every listed commercial region" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-07): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna" + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna" + "guardrails_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" } } }, @@ -368,31 +419,40 @@ "available": true, "model_id": "openai.gpt-5.4", "requires_cris": false, + "guardrails_supported": false, "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1, us-east-2, us-west-2 and us-gov-west-1; Geo/Global not supported on mantle" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": false, - "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code" + "guardrails_supported": false, + "capabilities": ["tool_or_function_calling", "reasoning", "streaming", "image_input_vision"], + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): structured outputs are evidenced on the Responses API only, so structured_output_json is not carried on Mantle Chat Completions" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": true, - "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.4 (Geo) or global.openai.gpt-5.4 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs" + "guardrails_supported": true, + "capabilities": ["tool_or_function_calling", "reasoning", "streaming", "image_input_vision"], + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.4 (Geo) or global.openai.gpt-5.4 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): 'Response streaming' listed for runtime; 'Structured outputs (JSON Schema; Responses API)' is scoped to Responses, so structured_output_json is not carried on Converse; Guardrails Converse-only" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-07): bedrock-runtime lists Responses supported; structured outputs and implicit prompt caching are Responses-only on runtime; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" + "guardrails_supported": false, + "capabilities": ["tool_or_function_calling", "structured_output_json", "reasoning", "streaming", "image_input_vision"], + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): bedrock-runtime lists Responses supported; 'Structured outputs (JSON Schema; Responses API)' and 'Response streaming' listed for runtime, implicit prompt caching Responses-only; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": true, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; the card scopes structured outputs and implicit prompt caching to Responses; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" + "guardrails_supported": false, + "capabilities": ["tool_or_function_calling", "reasoning", "streaming", "image_input_vision"], + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; 'Response streaming' listed for runtime; the card scopes 'Structured outputs (JSON Schema; Responses API)' and implicit prompt caching to Responses, so structured_output_json is not carried on Chat Completions; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" } } }, @@ -450,6 +510,7 @@ "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": false, + "guardrails_supported": false, "supported_regions": [ "us-east-1", "us-west-2" @@ -466,6 +527,7 @@ "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": false, + "guardrails_supported": false, "supported_regions": [ "us-east-1", "us-west-2" @@ -480,6 +542,7 @@ "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": true, + "guardrails_supported": true, "supported_regions": [ "us-east-1", "us-east-2", @@ -539,6 +602,7 @@ "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": true, + "guardrails_supported": false, "supported_regions": [ "us-east-1", "us-east-2", @@ -599,6 +663,7 @@ "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": true, + "guardrails_supported": false, "supported_regions": [ "us-east-1", "us-east-2", @@ -677,11 +742,11 @@ "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision"], "capability_source": "AWS GPT-6 Sol model card, retrieved 2026-10-10: text and image input, tool calling, 1.05M context and 128K output.", "paths": { - "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, - "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported."}, - "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-07): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"}, - "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"} + "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, + "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, + "runtime_converse": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported. (Guardrails: Responses/Chat Completions/InvokeModel/Converse per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html; card read 2026-11-08)"}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"} } }, "openai_gpt_6_luna": { @@ -704,11 +769,11 @@ "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision"], "capability_source": "AWS GPT-6 Luna model card, retrieved 2026-10-10: text and image input, tool calling, 1.05M context and 128K output.", "paths": { - "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, - "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke."}, - "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-07): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"}, - "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"} + "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, + "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, + "runtime_converse": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke. (Guardrails: Responses/Chat Completions/InvokeModel/Converse per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html; card read 2026-11-08)"}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"} } }, "openai_gpt_6_1_sol": { @@ -731,11 +796,11 @@ "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision"], "capability_source": "AWS GPT-6.1 Sol model card, retrieved 2026-10-10: text and image input, tool calling, 1.05M context and 131072 output.", "paths": { - "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, - "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported."}, - "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-07): bedrock-runtime lists Responses supported; Guardrails are listed for Chat Completions/InvokeModel/Converse but not Responses, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"}, - "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; Guardrails listed on Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"} + "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "guardrails_supported": false, "server_side_tools_supported": false, "evidence": "AWS model card: Mantle Responses in us-east-1. https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): Server-side system tools listed as Not Supported on bedrock-mantle"}, + "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "guardrails_supported": false, "server_side_tools_supported": false, "evidence": "AWS model card: Mantle Chat Completions supported. https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): Server-side system tools listed as Not Supported on bedrock-mantle"}, + "runtime_converse": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported. (Guardrails: Chat Completions/InvokeModel/Converse, not Responses, per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html; card read 2026-11-08)"}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": false, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails are listed for Chat Completions/InvokeModel/Converse but not Responses, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails listed on Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"} } }, "anthropic_claude_sonnet_5": { @@ -757,6 +822,7 @@ "available": true, "model_id": "anthropic.claude-sonnet-5", "requires_cris": true, + "guardrails_supported": true, "evidence": "AWS model card (2026-07-21): Converse on bedrock-runtime checked; Geo (us/eu/au) and Global CRIS supported" } } @@ -779,6 +845,7 @@ "available": true, "model_id": "anthropic.claude-haiku-4-5-20251001-v1:0", "requires_cris": true, + "guardrails_supported": true, "evidence": "anthropic-bedrock.json catalog (verified 2026-09-24) runtime_converse entry (requires CRIS profile)" } } @@ -872,7 +939,8 @@ "runtime_converse": { "available": true, "model_id": "anthropic.claude-opus-5-5", - "requires_cris": true + "requires_cris": true, + "guardrails_supported": true } }, "generation": "bedrock_native", From 250ff833ef0ef030869a56e53368658896099565 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 16:19:49 -0800 Subject: [PATCH 34/41] fix(aws-startup-advisor): agent-advisor engines validate selectors, Guardrails, logging and application profiles against the final model/API Engine behaviour (public pages read 2026-11-08): - CloudWatch never constrains the endpoint (Mantle publishes AWS/BedrockMantle metrics); only invocation logging, Guardrails, application inference profiles and multi-model Converse pin bedrock-runtime. - Invocation logging plus any Mantle selector (explicit, web_search-derived or derived in_region_processing) is a decision between the Mantle option (selector echoed) and the runtime option that keeps the source API, in both engines (OpenAI: model_path_decision_required; Anthropic: new conflicts mantle_selector_vs_invocation_logging / in_region_processing_vs_invocation_logging). - Guardrails force runtime_converse only when the source model's card scopes them to Converse for the source-preserving runtime OpenAI API; GPT-6 Sol / GPT-6 Luna keep Responses/Chat, GPT-6.1 Sol keeps Chat, every other GPT card and unrecognized sources stay Converse-only. Runtime candidate walks skip rows without guardrails_supported when Guardrails are required. - An explicit runtime preferred_api_path plus a selector (explicit or derived) is a preferred_api_path_conflict decision (OpenAI) / preferred_path_vs_mantle_selector, preferred_path_vs_converse, in_region_processing_vs_preferred_api_path (Anthropic). - mantle_selector background_inference on a Chat Completions Mantle path is a new selector_api_incompatible decision offering mantle_openai_responses vs runtime_openai_chat (inference-responses-api.html). - server_side_tools / web_search selectors skip Mantle rows flagged server_side_tools_supported=false; a GPT-6.1 Sol source falls to GPT-6 Astra with the selector retained and a fallback rationale. - An application-inference-profile ARN requires Converse: it is never emitted as the invocation id of a runtime_openai_* / mantle_* option, and the verifier returns needs_resolution / ApplicationProfileUnsupportedOnApi for such an ARN on runtime_openai_responses|chat without constructing a client. - The Astra non-streaming JSON Schema guard covers runtime_converse, runtime_openai_responses and runtime_openai_chat (model-card-openai-gpt-6-astra.html). - Recommended records assert a set selector only ever lands on a Mantle path. Tests updated for the new behaviour: GPT-6 Sol/Luna + Guardrails now stay on runtime_openai_responses; GPT-6 Sol + Guardrails from eu-west-1 fails closed (inference_profile_unverified) instead of switching families; GPT-6.1 Sol + server_side_tools falls to Astra; the Mantle Chat verifier probes use the projects selector. Verification: - skills/agent-advisor/scripts: uv run --group dev pytest -q -> 786 passed (was 742) - plugins/aws-startup-advisor: mise run test -> 1495 passed + 222 subtests, 1 pre-existing failure tests/test_model_id_lint.py::test_real_plugin_passes (untracked skills/llm-to-bedrock/scripts/.venv; unrelated) - skills/llm-to-bedrock/scripts: uv run --group dev pytest -q -> 655 passed (no files changed) - python3 tools/sync-vendored.py --check -> PASS (65 files) - git diff --check -> clean --- .../scripts/anthropic_model_recommendation.py | 54 +- .../scripts/openai_model_recommendation.py | 418 ++++++++++++--- .../scripts/test_model_recommendation.py | 95 ++++ .../test_openai_model_recommendation.py | 483 +++++++++++++++++- .../scripts/test_verify_model_path.py | 33 +- .../scripts/verify_model_path.py | 15 + 6 files changed, 1005 insertions(+), 93 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/anthropic_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/anthropic_model_recommendation.py index 5b02be7de..1e10516d8 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/anthropic_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/anthropic_model_recommendation.py @@ -253,7 +253,13 @@ def _path_constraints(workload): and requirements.get("preserve_messages_api") is not False ) + # Model invocation logging is captured only on bedrock-runtime + # (model-invocation-logging.html, read 2026-11-08); CloudWatch metrics exist on + # both endpoints (AWS/BedrockMantle) and never constrain the path. + logging_required = "invocation_logging" in requirements.get("governance", []) + runtime_path = "runtime_invoke" if native_required else "runtime_converse" + messages_option = "runtime_messages" if (source_messages or messages_required) else runtime_path conflicts = [] option_paths = [] if messages_required and (converse_required or native_required): @@ -262,10 +268,19 @@ def _path_constraints(workload): if selector and (converse_required or native_required): conflicts.append("mantle_selector_vs_runtime") option_paths = ["mantle_messages", runtime_path] + elif selector and logging_required: + conflicts.append("mantle_selector_vs_invocation_logging") + option_paths = ["mantle_messages", messages_option] if preferred: if preferred.startswith("mantle_") and (converse_required or native_required): conflicts.append("preferred_path_vs_runtime") option_paths = option_paths or ["mantle_messages", runtime_path] + if preferred.startswith("runtime_") and selector: + conflicts.append("preferred_path_vs_mantle_selector") + option_paths = option_paths or ["mantle_messages", preferred] + if preferred == "runtime_messages" and converse_required: + conflicts.append("preferred_path_vs_converse") + option_paths = option_paths or ["runtime_messages", runtime_path] if preferred in {"runtime_converse", "runtime_invoke"} and messages_required: conflicts.append("preferred_path_vs_messages") option_paths = option_paths or ["runtime_messages", runtime_path] @@ -483,9 +498,10 @@ def _resolve_invocation_model_id(model_id, requires_cris, requirements, model=No return candidate if profile_id in allowed else None -def _decision_options(catalog, workload, option_paths, region=None): +def _decision_options(catalog, workload, option_paths, region=None, selector=None): requirements = workload["requirements"] - selector = requirements.get("mantle_selector") + if selector is None: + selector = requirements.get("mantle_selector") options = [] for path in dict.fromkeys(option_paths): candidates = _build_candidates(catalog, [path], requirements, region) @@ -493,8 +509,8 @@ def _decision_options(catalog, workload, option_paths, region=None): continue if path == "runtime_messages": tradeoff = ( - "Keeps the Anthropic Messages API on bedrock-runtime (IAM, invocation logging, " - "CloudWatch); Converse-only features such as Guardrails and the multi-model " + "Keeps the Anthropic Messages API on bedrock-runtime (IAM, invocation " + "logging); Converse-only features such as Guardrails and the multi-model " "surface are not on this API." ) elif path == "mantle_messages": @@ -792,7 +808,8 @@ def _verification(candidate, region, catalog, invocation_model_id, requirements= def _decision_required(workload, region, catalog, constraints): feature_status = _feature_assessment(workload, _source_version(workload["source"]), None) decision_options = _decision_options( - catalog, workload, constraints["option_paths"], region + catalog, workload, constraints["option_paths"], region, + selector=constraints.get("mantle_selector"), ) if len(decision_options) != len(set(constraints["option_paths"])): raise ValueError( @@ -883,13 +900,34 @@ def recommend_anthropic_workload(workload, region, catalog): requirements = workload["requirements"] rationale = list(constraints["rationale"]) mantle_selector = constraints["mantle_selector"] if path == "mantle_messages" else None - if ( + logging_required = "invocation_logging" in requirements.get("governance", []) + runtime_pinned = _converse_required(requirements) or requirements.get("requires_native_payload") + derivable = ( requirements.get("data_residency") == "in_region_required" and path.startswith("runtime_") and path_config["requires_cris"] and not _runtime_in_region_allowed(path_config["model_id"], model, region, requirements) - and not (_converse_required(requirements) or requirements.get("requires_native_payload")) - ): + ) + in_region_conflict = None + if derivable and requirements.get("preferred_api_path"): + in_region_conflict = "in_region_processing_vs_preferred_api_path" + elif derivable and logging_required and not runtime_pinned: + in_region_conflict = "in_region_processing_vs_invocation_logging" + if in_region_conflict and _build_candidates(catalog, ["mantle_messages"], requirements, region): + # Derived in_region_processing conflicts with an explicit runtime preference or + # with invocation logging (captured only on bedrock-runtime): the user decides. + return _decision_required(workload, region, catalog, { + "paths": [], + "conflicts": [in_region_conflict], + "option_paths": ["mantle_messages", path], + "rationale": [ + "Strict in-Region processing was requested and bedrock-runtime serves this " + "model only through a CRIS profile here (derived mantle_selector " + "in_region_processing); the recorded requirements cannot share one path." + ], + "mantle_selector": "in_region_processing", + }) + if derivable and not runtime_pinned: # Derived selector (bedrock-endpoint-policy.md in_region_processing): bedrock-runtime # serves this model here only through a CRIS profile, which may route out of the Region. in_region = _build_candidates(catalog, ["mantle_messages"], requirements, region) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py index d8bf468d3..a2d04f01d 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py @@ -209,14 +209,77 @@ def _api_label(path): return "Chat Completions" if path.endswith("_chat") else "Responses" -def _converse_required(requirements): - """Converse is the only API every card evidences for Guardrails / application - inference profiles; multi-model Converse is Converse by definition.""" - return "guardrails" in (requirements.get("governance") or []) or bool( - requirements.get("multi_model_converse", False) +def _logging_required(requirements): + """Model invocation logging is captured only on bedrock-runtime (including its + OpenAI-compatible APIs): model-invocation-logging.html, read 2026-11-08. CloudWatch + metrics exist on both endpoints (AWS/BedrockMantle) and never constrain the path.""" + return "invocation_logging" in (requirements.get("governance") or []) + + +def _guardrails_required(requirements): + return "guardrails" in (requirements.get("governance") or []) + + +def _application_profile_requested(requirements): + """An application inference profile ARN is accepted by Converse only; the + bedrock-runtime Responses and Chat Completions APIs reject it with HTTP 400 + (inference-responses-api.html, read 2026-11-08).""" + return ":application-inference-profile/" in (requirements.get("inference_profile_id") or "") + + +def _guardrails_on_openai_api(catalog, source, openai_path): + """True iff the recognized source model's catalog row for `openai_path` lists + Guardrails on that API (per model card). Unrecognized sources keep Converse.""" + same = _same_model_catalog_key(source) + if not same or not openai_path or catalog is None: + return False + row = catalog["models"].get(same, {}).get("paths", {}).get(openai_path, {}) + return row.get("guardrails_supported") is True + + +def _converse_required(requirements, catalog=None, source=None, openai_path=None): + """Converse is required by multi-model Converse, by an application inference + profile, and by Guardrails when the source model's card scopes them to Converse + (the GPT-6 Sol / GPT-6 Luna / GPT-6.1 Sol cards list Guardrails on some runtime + OpenAI APIs, so those sources keep their API).""" + return ( + bool(requirements.get("multi_model_converse", False)) + or _application_profile_requested(requirements) + or ( + _guardrails_required(requirements) + and not _guardrails_on_openai_api(catalog, source, openai_path) + ) ) +def _converse_only_drivers(requirements, guardrails_on_api=False): + """Human-readable list of the requirements that pin this workload to Converse.""" + drivers = [] + if _guardrails_required(requirements) and not guardrails_on_api: + drivers.append("Guardrails, which the model card lists as Converse-only for this model") + if _application_profile_requested(requirements): + drivers.append( + "the application inference profile, which bedrock-runtime accepts only on Converse" + ) + if requirements.get("multi_model_converse"): + drivers.append("multi-model Converse") + return drivers + + +def _runtime_only_drivers(requirements): + """Human-readable list of the governance requirements that pin bedrock-runtime.""" + drivers = [] + if _guardrails_required(requirements): + drivers.append("Guardrails") + if _logging_required(requirements): + drivers.append("invocation logging") + if _application_profile_requested(requirements): + drivers.append("application inference profiles") + if requirements.get("multi_model_converse"): + drivers.append("multi-model Converse") + return drivers + + def _mantle_selector(workload, feature_assessment): """Explicit requirements.mantle_selector, else the derived web_search selector. in_region_processing is derived later, once the runtime candidate is known.""" @@ -276,13 +339,21 @@ def _hard_profile_requirement(requirements): ) +_SERVER_SIDE_TOOL_SELECTORS = frozenset({"server_side_tools", "web_search"}) + + def _catalog_model_for_path(catalog, path, detected_features=None, requirements=None, - candidate_order=None, region=None): + candidate_order=None, region=None, selector=None, + enforce_guardrails=True): """Pick the first available catalog model for `path` whose capability evidence covers the required/detected text features (evidence-driven fallback: a newer model without feature-level evidence is skipped in favor of an older probed one). Mantle candidates with cataloged region availability are skipped outside that footprint; runtime Converse availability remains governed by CRIS resolution. + When Guardrails are required, runtime rows whose card does not list Guardrails on + that API are skipped (`enforce_guardrails=False` lets a decision option show the + API that gives Guardrails up). A server-side-tool selector skips Mantle rows whose + card lists server-side tools as unsupported (GPT-6.1 Sol). When no candidate covers the requirements, return the first available model together with its unmet-capability list so the caller can fail closed. Deterministic: iterates `candidate_order` when given (e.g. the Converse tier map), else @@ -290,6 +361,7 @@ def _catalog_model_for_path(catalog, path, detected_features=None, requirements= first = None first_unmet = None keys = candidate_order if candidate_order is not None else list(catalog["models"]) + guardrails = enforce_guardrails and _guardrails_required(requirements or {}) for model_key in keys: model = catalog["models"].get(model_key) if model is None: @@ -297,6 +369,18 @@ def _catalog_model_for_path(catalog, path, detected_features=None, requirements= path_config = model["paths"].get(path, {}) if path_config.get("available") is not True: continue + if ( + guardrails + and path.startswith("runtime_") + and path_config.get("guardrails_supported") is not True + ): + continue + if ( + selector in _SERVER_SIDE_TOOL_SELECTORS + and path.startswith("mantle_") + and path_config.get("server_side_tools_supported") is False + ): + continue if "capabilities" in path_config: model = {**model, "capabilities": path_config["capabilities"]} availability = model.get("region_availability") @@ -329,7 +413,8 @@ def _catalog_model_for_path(catalog, path, detected_features=None, requirements= _PROFILE_REQUIREMENT_KEYS = frozenset({"inference_profile_id", "data_residency", "cris_geography"}) -def _profile_relaxed_hit(catalog, path, detected, requirements, candidate_order, region): +def _profile_relaxed_hit(catalog, path, detected, requirements, candidate_order, region, + enforce_guardrails=True): """When every runtime candidate was skipped because no cataloged CRIS profile is callable from `region` under the stated residency, re-walk the path without the profile requirements so a decision can still show the candidate. The caller @@ -339,6 +424,7 @@ def _profile_relaxed_hit(catalog, path, detected, requirements, candidate_order, relaxed = {k: v for k, v in (requirements or {}).items() if k not in _PROFILE_REQUIREMENT_KEYS} hit, _ = _catalog_model_for_path( catalog, path, detected, relaxed, candidate_order=candidate_order, region=region, + enforce_guardrails=enforce_guardrails, ) return hit @@ -397,9 +483,14 @@ def _api_continuity(requirements): return "unknown" -def _runtime_required(requirements): - return bool(requirements.get("governance")) or requirements.get( - "multi_model_converse", False +def _runtime_required(requirements, converse_required): + """bedrock-runtime is pinned by anything Converse-only, by invocation logging + (runtime-only capture) and by Guardrails (no Mantle row lists them). CloudWatch + metrics exist on both endpoints and never pin the path.""" + return bool( + converse_required + or _logging_required(requirements) + or _guardrails_required(requirements) ) @@ -972,10 +1063,12 @@ def _verification(region, catalog, path, requires_cris, invocation_model_id, sel } -def _decision_options(catalog, workload, region, selector=None): +def _decision_options(catalog, workload, region, selector=None, openai_path=None): """Two-sided option set. Option A preserves the OpenAI SDK and the source API: the Mantle path when a mantle_selector exists, else the bedrock-runtime - OpenAI path. Option B is runtime Converse governance.""" + OpenAI path (`openai_path` overrides that choice). Option B is runtime Converse + governance. An application inference profile is never emitted as the invocation + id of an OpenAI-API option: bedrock-runtime Responses/Chat reject it with HTTP 400.""" options = [] source = workload["source"] requirements = workload["requirements"] @@ -983,7 +1076,12 @@ def _decision_options(catalog, workload, region, selector=None): if status == "detected"] astra = _same_model_runtime_key(source) == "openai_gpt_6_astra" surface = source.get("api_surface") - if selector: + application_profile = _application_profile_requested(requirements) + if openai_path: + # Caller-fixed option path (an explicit runtime preference, or the Mantle + # Responses reshape offered for background inference). + pass + elif selector: openai_path = _mantle_path(source, requirements) if astra: preferred = requirements.get("preferred_api_path") @@ -1001,21 +1099,25 @@ def _decision_options(catalog, workload, region, selector=None): candidate_order = ( ["openai_gpt_6_astra"] if astra else _openai_candidate_order(catalog, source) ) + # The OpenAI-API option is the side of the decision that may give Guardrails up, + # so the walk shows the source-preserving model rather than skipping it. openai_hit, _ = _catalog_model_for_path( catalog, openai_path, detected, requirements, - candidate_order=candidate_order, region=region, + candidate_order=candidate_order, region=region, selector=selector, + enforce_guardrails=False, ) requires_region_change = False if openai_hit is None and selector and region != "unknown": openai_hit, _ = _catalog_model_for_path( catalog, openai_path, detected, requirements, candidate_order=_openai_candidate_order(catalog, source), - region="unknown", + region="unknown", selector=selector, enforce_guardrails=False, ) requires_region_change = openai_hit is not None if openai_hit is None and openai_path.startswith("runtime_openai_"): openai_hit = _profile_relaxed_hit( - catalog, openai_path, detected, requirements, candidate_order, region + catalog, openai_path, detected, requirements, candidate_order, region, + enforce_guardrails=False, ) if openai_hit: model_key, model, path_config = openai_hit @@ -1024,17 +1126,23 @@ def _decision_options(catalog, workload, region, selector=None): if openai_path in {"mantle_openai_chat", "runtime_openai_chat"} else "Responses" ) - if selector: + if openai_path.startswith("mantle_"): reason = ( f"Preserves the OpenAI SDK and the {api_label} surface on bedrock-mantle, " f"selected by mantle_selector {selector}; gives up bedrock-runtime-only " - "governance (Guardrails, invocation logging)." + "governance (Guardrails, invocation logging, application inference profiles)." + ) + elif _guardrails_required(requirements) and path_config.get("guardrails_supported") is True: + reason = ( + f"Keeps the OpenAI SDK and the {api_label} API on bedrock-runtime via a CRIS id " + "(invocation logging included); Guardrails are listed on this API for this " + "model (see its model card)." ) else: reason = ( "Keeps the OpenAI SDK and the source API on bedrock-runtime via a CRIS id " - "(invocation logging and CloudWatch included); Guardrails and application " - "inference profiles are evidenced only on Converse for this model — see its card." + "(invocation logging included); Guardrails and application inference " + "profiles are evidenced only on Converse for this model — see its card." ) if requires_region_change: availability = model.get("region_availability") or {} @@ -1047,12 +1155,22 @@ def _decision_options(catalog, workload, region, selector=None): f" Requires changing from {region} to a catalog-supported Mantle region " f"({region_label})." ) - invocation_model_id = _resolve_invocation_model_id( - path_config["model_id"], path_config["requires_cris"], requirements, model, region, - ) - reason += _unresolved_profile_suffix( - openai_path, path_config, invocation_model_id, requirements, region - ) + if application_profile: + invocation_model_id = None + reason += ( + " Application inference profiles are rejected with HTTP 400 on the " + "bedrock-runtime Responses and Chat Completions APIs " + "(https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, " + "read 2026-11-08); Converse is the supported API for this profile." + ) + else: + invocation_model_id = _resolve_invocation_model_id( + path_config["model_id"], path_config["requires_cris"], requirements, model, + region, + ) + reason += _unresolved_profile_suffix( + openai_path, path_config, invocation_model_id, requirements, region + ) options.append( { "model_key": model_key, @@ -1082,14 +1200,24 @@ def _decision_options(catalog, workload, region, selector=None): path_config["model_id"], path_config["requires_cris"], workload["requirements"], model, region, ) - reason = ( - "SAME-MODEL governance path: this OpenAI target runs on bedrock-runtime via a " - "CRIS id — Guardrails (Converse API only) and invocation logging without a " - "model change. Check the model's region/profile matrix and dated prices." - if model_key == _same_model_runtime_key(workload["source"]) - else "Uses Bedrock-native Converse request/response shapes and a " - "cross-family model; requires rewriting the OpenAI integration." + same_model = model_key == _same_model_runtime_key(workload["source"]) + guardrails_on_openai_api = _guardrails_on_openai_api( + catalog, source, + openai_path if openai_path.startswith("runtime_openai_") else _runtime_openai_path(surface), ) + if same_model: + reason = ( + "SAME-MODEL governance path: this OpenAI target runs on bedrock-runtime via a " + "CRIS id — " + + ("Guardrails" if guardrails_on_openai_api else "Guardrails (Converse API only)") + + ", invocation logging and application inference profiles without a " + "model change. Check the model's region/profile matrix and dated prices." + ) + else: + reason = ( + "Uses Bedrock-native Converse request/response shapes and a " + "cross-family model; requires rewriting the OpenAI integration." + ) reason += _unresolved_profile_suffix( "runtime_converse", path_config, invocation_model_id, workload["requirements"], region, @@ -1125,8 +1253,19 @@ def recommend_openai_workload(workload, region, catalog): requirements = workload["requirements"] surface = source.get("api_surface") continuity = _api_continuity(requirements) - runtime_required = _runtime_required(requirements) - converse_required = _converse_required(requirements) + preferred = requirements.get("preferred_api_path") + # The bedrock-runtime OpenAI API this workload would keep: an explicit runtime + # OpenAI preference, else the API that preserves the source surface. Guardrails + # are checked against the source model's row for exactly this API. + openai_path_default = ( + preferred if (preferred or "").startswith("runtime_openai_") + else _runtime_openai_path(surface) + ) + converse_required = _converse_required(requirements, catalog, source, openai_path_default) + runtime_required = _runtime_required(requirements, converse_required) + guardrails_on_api = _guardrails_required(requirements) and _guardrails_on_openai_api( + catalog, source, openai_path_default + ) continuity_required = continuity == "required" feature_assessment = _feature_assessment(workload) @@ -1207,28 +1346,65 @@ def _decision_conflict(reason_head, reasons, message, remediation, options): "", decision_options=options, ) + def _mantle_options(mantle_selector): + return [ + option for option in _decision_options(catalog, workload, region, mantle_selector) + if option["api_path"].startswith("mantle_") + ] + + def _runtime_option(runtime_path): + # The single bedrock-runtime option that keeps `runtime_path`. + if runtime_path == "runtime_converse": + candidates = _decision_options(catalog, workload, region, None) + else: + candidates = _decision_options( + catalog, workload, region, None, openai_path=runtime_path + ) + return [option for option in candidates if option["api_path"] == runtime_path][:1] + + def _preferred_path_conflict(mantle_selector, message): + # An explicit bedrock-runtime preference cannot carry a Mantle selector + # (explicit, web_search-derived or derived in_region_processing): the user + # chooses between the Mantle option and the preferred runtime API. + return _unresolved( + "The explicit bedrock-runtime API preference conflicts with the recorded " + "Mantle selector.", + [], + _finding( + "preferred_api_path_conflict", "[BLOCKS]", message, + "Keep the preference and drop the selector, or accept the Mantle option, " + "then rerun Model Recommend.", + ), + preferred, + decision_options=_mantle_options(mantle_selector) + _runtime_option(preferred), + ) + # --- Hard conflict: a verified Mantle-only need AND bedrock-runtime-only governance --- if selector and runtime_required: - options = _decision_options(catalog, workload, region, selector) - if not converse_required: - # Logging/CloudWatch governance is satisfied on any bedrock-runtime path, so - # the runtime option keeps the OpenAI API rather than switching to Converse. - options = [ - option for option in options if option["api_path"].startswith("mantle_") - ] + [ - option for option in _decision_options(catalog, workload, region, None) + drivers = ", ".join(_runtime_only_drivers(requirements)) + if converse_required: + options = _decision_options(catalog, workload, region, selector) + else: + # Invocation logging and Guardrails listed on the OpenAI API are satisfied + # on the bedrock-runtime OpenAI path, so the runtime option keeps the + # source API rather than switching to Converse. + options = _mantle_options(selector) + [ + option for option in _decision_options( + catalog, workload, region, None, openai_path=openai_path_default + ) if option["api_path"].startswith("runtime_openai_") ] return _decision_conflict( f"A verified Mantle-only need (mantle_selector {selector}) conflicts with a " - "bedrock-runtime-only governance requirement.", + f"bedrock-runtime-only governance requirement ({drivers}).", [ - "bedrock-mantle serves the selected capability; bedrock-runtime provides " - "Guardrails, invocation logging and application inference profiles " - "(bedrock-endpoint-policy.md)." + "bedrock-mantle serves the selected capability; bedrock-runtime alone " + "provides Guardrails, invocation logging and application inference " + "profiles (bedrock-endpoint-policy.md; model-invocation-logging.html, read " + "2026-11-08)." ], f"A verified Mantle-only need (mantle_selector {selector}) and " - "bedrock-runtime-only governance cannot both be satisfied by one path.", + f"bedrock-runtime-only governance ({drivers}) cannot both be satisfied by one path.", "Choose the Mantle option or the bedrock-runtime governance option, update " "requirements, and rerun Model Recommend.", options, @@ -1236,20 +1412,21 @@ def _decision_conflict(reason_head, reasons, message, remediation, options): # --- Hard conflict: OpenAI continuity required AND Converse-only governance --- if continuity_required and converse_required: + drivers = "; ".join(_converse_only_drivers(requirements, guardrails_on_api)) return _decision_conflict( - "Required OpenAI API continuity conflicts with Converse-only governance " - "(Guardrails / multi-model Converse).", + f"Required OpenAI API continuity conflicts with Converse-only governance ({drivers}).", [ "bedrock-runtime Responses/Chat Completions keep the OpenAI SDK and the " - "source API via a CRIS id; runtime Converse provides Guardrails and the " - "multi-model surface and keeps an exact same-model runtime candidate when " - "the catalog provides one." + "source API via a CRIS id; runtime Converse provides Guardrails (where the " + "model card lists them as Converse-only), application inference profiles " + "and the multi-model surface and keeps an exact same-model runtime candidate " + "when the catalog provides one." ], - "Required OpenAI API continuity and Converse-only governance (Guardrails / " - "multi-model Converse) cannot both be satisfied by one bedrock-runtime API.", + f"Required OpenAI API continuity and Converse-only governance ({drivers}) " + "cannot both be satisfied by one bedrock-runtime API.", "Choose the bedrock-runtime OpenAI API option or the runtime Converse " "governance option, update requirements, and rerun Model Recommend.", - _decision_options(catalog, workload, region, None), + _decision_options(catalog, workload, region, None, openai_path=openai_path_default), ) # --- Select a path --- @@ -1279,6 +1456,16 @@ def _decision_conflict(reason_head, reasons, message, remediation, options): f"Mantle {_api_label(path)} selected by mantle_selector {selector} " "(bedrock-endpoint-policy.md); the exact source model is evaluated first." ) + elif guardrails_on_api: + path = openai_path_default + candidate_order = _openai_candidate_order(catalog, source) + same_name = catalog["models"][_same_model_catalog_key(source)]["display_name"] + rationale_head = ( + f"Guardrails are listed on {_api_label(path)} for {same_name} (model card, read " + "2026-11-08), so the source API stays on bedrock-runtime; the exact source " + "model is evaluated first and only candidates whose card lists Guardrails on " + "this API are considered." + ) else: path = _runtime_openai_path(surface) candidate_order = _openai_candidate_order(catalog, source) @@ -1298,7 +1485,6 @@ def _decision_conflict(reason_head, reasons, message, remediation, options): "otherwise the source-compatible API or required runtime Converse path; " "apply Astra's own region and feature evidence." ) - preferred = requirements.get("preferred_api_path") if preferred: first = catalog["models"].get(candidate_order[0], {}) if candidate_order else {} available = first.get("paths", {}).get(preferred, {}) @@ -1318,20 +1504,59 @@ def _decision_conflict(reason_head, reasons, message, remediation, options): "(bedrock-endpoint-policy.md)." if preferred.startswith("mantle_") and not selector else "" + ) + + ( + " The requested application inference profile is accepted only on " + "Converse; the bedrock-runtime Responses and Chat Completions APIs " + "reject it with HTTP 400 (inference-responses-api.html, read 2026-11-08)." + if _application_profile_requested(requirements) + and preferred.startswith("runtime_openai_") + else "" ), "Choose a supported API or revise the conflicting requirements, " "then rerun Model Recommend.", ), preferred, ) + if preferred.startswith("runtime_") and selector: + return _preferred_path_conflict( + selector, + f"The requested {preferred} path cannot satisfy mantle_selector {selector}; " + "that capability is served on bedrock-mantle only (bedrock-endpoint-policy.md).", + ) path = preferred + # Background inference is a Responses API feature on bedrock-mantle; a Chat + # Completions Mantle path cannot carry it, so the reshape is offered as a + # decision rather than selected silently. + if selector == "background_inference" and path == "mantle_openai_chat": + return _unresolved( + "mantle_selector background_inference conflicts with the Chat Completions API.", + [], + _finding( + "selector_api_incompatible", "[BLOCKS]", + "Background inference is a Responses API feature on bedrock-mantle; Chat " + "Completions has no background mode " + "(https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, " + "read 2026-11-08).", + "Accept the Mantle Responses reshape, or keep Chat Completions on " + "bedrock-runtime and drop background inference, then rerun Model Recommend.", + ), + path, + decision_options=[ + option for option in _decision_options( + catalog, workload, region, selector, openai_path="mantle_openai_responses" + ) + if option["api_path"] == "mantle_openai_responses" + ] + _runtime_option("runtime_openai_chat"), + ) + # Evidence-driven selection: pick the first candidate on the path whose catalog # capabilities cover the required/detected text features (a newer model without # feature-level evidence falls back to an older probed one). G02: fail closed # only when NO candidate on the path has the evidence. catalog_hit, unsupported = _catalog_model_for_path( catalog, path, detected, requirements, candidate_order=candidate_order, - region=region, + region=region, selector=selector, ) if not catalog_hit: if path.startswith("mantle_"): @@ -1379,23 +1604,50 @@ def _decision_conflict(reason_head, reasons, message, remediation, options): model_key, model, path_config = catalog_hit # Derived selector: strict in-Region processing on a CRIS-only runtime candidate - # is a Mantle-only need (bedrock-endpoint-policy.md in_region_processing), unless - # bedrock-runtime-only governance pins the workload to runtime. - if ( + # is a Mantle-only need (bedrock-endpoint-policy.md in_region_processing). An + # explicit runtime preference or invocation logging turns it into a decision; + # Converse-only governance keeps the runtime path (and its residency block). + derivable = ( path.startswith("runtime_") and requirements.get("data_residency") == "in_region_required" and path_config["requires_cris"] and not anthropic_model_recommendation._runtime_in_region_allowed( path_config["model_id"], model, region, requirements ) - and not runtime_required - ): + ) + if derivable and preferred and preferred.startswith("runtime_"): + return _preferred_path_conflict( + "in_region_processing", + f"The requested {preferred} path serves this model only through a CRIS profile, " + "which conflicts with data_residency in_region_required (derived mantle_selector " + "in_region_processing).", + ) + if derivable and _logging_required(requirements) and not converse_required: + return _decision_conflict( + "Strict in-Region processing (derived mantle_selector in_region_processing) " + "conflicts with invocation logging, which bedrock-runtime alone provides.", + [ + f"bedrock-runtime serves this model from {region} only through a CRIS " + "profile; model invocation logging is captured only on bedrock-runtime " + "(model-invocation-logging.html, read 2026-11-08)." + ], + "in_region_processing and invocation logging cannot both be satisfied by one path.", + "Choose the Mantle in-Region option or the bedrock-runtime logging option, " + "update requirements, and rerun Model Recommend.", + _mantle_options("in_region_processing") + [ + option for option in _decision_options( + catalog, workload, region, None, openai_path=openai_path_default + ) + if option["api_path"].startswith("runtime_openai_") + ], + ) + if derivable and not runtime_required: selector = "in_region_processing" path = _mantle_path(source, requirements) candidate_order = ["openai_gpt_6_astra"] if astra else _openai_candidate_order(catalog, source) catalog_hit, unsupported = _catalog_model_for_path( catalog, path, detected, requirements, candidate_order=candidate_order, - region=region, + region=region, selector=selector, ) if not catalog_hit: return _unresolved( @@ -1438,6 +1690,24 @@ def _decision_conflict(reason_head, reasons, message, remediation, options): f"The exact source model is unavailable in {region}; " f"{model['display_name']} is the in-region {endpoint} Chat Completions fallback." ) + same_key = _same_model_catalog_key(source) + if ( + path.startswith("mantle_") + and selector in _SERVER_SIDE_TOOL_SELECTORS + and same_key + and model_key != same_key + and catalog["models"][same_key]["paths"].get(path, {}).get("server_side_tools_supported") + is False + ): + # The selector is never dropped: the source model's card lists server-side + # system tools as unsupported on bedrock-mantle, so the same-family walk + # supplies the model that serves them. + rationale_head = ( + f"{catalog['models'][same_key]['display_name']} lists server-side system tools as " + "unsupported on bedrock-mantle (model card, read 2026-11-08); " + f"{model['display_name']} is the same-family Mantle {_api_label(path)} fallback " + f"that serves mantle_selector {selector}." + ) if unsupported: return _unresolved( rationale_head, @@ -1457,13 +1727,17 @@ def _decision_conflict(reason_head, reasons, message, remediation, options): ) needed = set(detected) | set(requirements.get("critical_features") or []) - if (model_key == "openai_gpt_6_astra" and path == "runtime_converse" + if (model_key == "openai_gpt_6_astra" + and path in {"runtime_converse", "runtime_openai_responses", "runtime_openai_chat"} and {"structured_output_json", "streaming"} <= needed): return _unresolved( rationale_head, [], _finding( "structured_output_streaming_unverified", "[BLOCKS]", - "Astra runtime JSON Schema output is documented only for non-streaming calls.", + "Astra JSON Schema output on bedrock-runtime applies to non-streaming calls " + "on Chat Completions, Responses and Converse " + "(https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html, " + "read 2026-11-08).", "Use non-streaming structured output or separate the streaming workload, " "then update requirements and rerun Model Recommend.", ), path, @@ -1630,6 +1904,18 @@ def _decision_conflict(reason_head, reasons, message, remediation, options): f"{model['display_name']} path is evidenced by the reference, but its context/output " "limits are unknown in the catalog and must be probed." ) + # Invariants: a set selector is only ever satisfied on a Mantle path (every other + # combination returned a decision above), and a runtime path carries no selector. + if selector and not path.startswith("mantle_"): + raise ValueError( + f"mantle_selector {selector} cannot be recommended on {path} for workload " + f"{workload['workload_id']}" + ) + if _application_profile_requested(requirements) and path != "runtime_converse": + raise ValueError( + f"an application inference profile cannot be recommended on {path} for workload " + f"{workload['workload_id']}" + ) rec = _base(workload, "recommended") rec.update( { diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_model_recommendation.py index 5ffb47166..1c4162a50 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_model_recommendation.py @@ -187,6 +187,101 @@ def test_selector_and_guardrails_require_user_decision_with_mantle_option(): assert "mantle_selector_vs_runtime" in block["message"] +def _decision_block(rec): + return next(item for item in rec["blocks"] if item["code"] == "model_path_decision_required") + + +# Invocation logging is captured only on bedrock-runtime (model-invocation-logging.html, +# read 2026-11-08); CloudWatch metrics exist on both endpoints (AWS/BedrockMantle, +# monitoring-mantle-metrics.html, read 2026-11-08) and never constrain the path. + +def test_logging_plus_selector_is_a_decision_between_mantle_and_runtime_messages(): + workload = _workload(requirements={"governance": ["invocation_logging"], + "mantle_selector": "workspaces"}) + result = model_recommendation.recommend(_input(workload)) + jsonschema.validate(result, _schema("model-recommendation.json")) + rec = result["workloads"]["support-agent"] + + assert rec["decision_status"] == "decision_required" + options = {option["api_path"]: option for option in rec["decision_options"]} + assert set(options) == {"mantle_messages", "runtime_messages"} + assert options["mantle_messages"]["mantle_selector"] == "workspaces" + assert "mantle_selector_vs_invocation_logging" in _decision_block(rec)["message"] + assert "CloudWatch" not in _decision_block(rec)["message"] + + +def test_logging_plus_derived_in_region_is_a_decision_carrying_the_derived_selector(): + workload = _workload(requirements={"governance": ["invocation_logging"], + "data_residency": "in_region_required"}) + result = model_recommendation.recommend(_input(workload)) + jsonschema.validate(result, _schema("model-recommendation.json")) + rec = result["workloads"]["support-agent"] + + assert rec["decision_status"] == "decision_required" + options = {option["api_path"]: option for option in rec["decision_options"]} + assert set(options) == {"mantle_messages", "runtime_messages"} + assert options["mantle_messages"]["mantle_selector"] == "in_region_processing" + assert "in_region_processing_vs_invocation_logging" in _decision_block(rec)["message"] + assert "runtime_messages" in _decision_block(rec)["remediation"] + + +def test_cloudwatch_plus_selector_is_recommended_on_mantle_messages(): + rec = _recommend(_workload(requirements={"governance": ["cloudwatch"], + "mantle_selector": "workspaces"})) + + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "mantle_messages" + assert rec["mantle_selector"] == "workspaces" + + +def test_cloudwatch_plus_in_region_derives_the_mantle_selector(): + rec = _recommend(_workload(requirements={"governance": ["cloudwatch"], + "data_residency": "in_region_required"})) + + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "mantle_messages" + assert rec["mantle_selector"] == "in_region_processing" + + +# An explicit bedrock-runtime preference never coexists with a Mantle selector or a +# Converse-only requirement; both are decisions. + +def test_preferred_runtime_messages_plus_selector_is_a_decision(): + rec = _recommend(_workload(requirements={"preferred_api_path": "runtime_messages", + "mantle_selector": "workspaces"})) + + assert rec["decision_status"] == "decision_required" + options = {option["api_path"]: option for option in rec["decision_options"]} + assert set(options) == {"mantle_messages", "runtime_messages"} + assert options["mantle_messages"]["mantle_selector"] == "workspaces" + assert "preferred_path_vs_mantle_selector" in _decision_block(rec)["message"] + + +def test_preferred_runtime_messages_plus_guardrails_is_a_decision(): + rec = _recommend(_workload(requirements={"preferred_api_path": "runtime_messages", + "governance": ["guardrails"]})) + + assert rec["decision_status"] == "decision_required" + assert {option["api_path"] for option in rec["decision_options"]} == { + "runtime_messages", "runtime_converse", + } + assert "preferred_path_vs_converse" in _decision_block(rec)["message"] + + +def test_preferred_runtime_messages_plus_derived_in_region_is_a_decision(): + workload = _workload(requirements={"preferred_api_path": "runtime_messages", + "data_residency": "in_region_required"}) + result = model_recommendation.recommend(_input(workload)) + jsonschema.validate(result, _schema("model-recommendation.json")) + rec = result["workloads"]["support-agent"] + + assert rec["decision_status"] == "decision_required" + options = {option["api_path"]: option for option in rec["decision_options"]} + assert set(options) == {"mantle_messages", "runtime_messages"} + assert options["mantle_messages"]["mantle_selector"] == "in_region_processing" + assert "in_region_processing_vs_preferred_api_path" in _decision_block(rec)["message"] + + def test_in_region_required_on_a_cris_only_runtime_candidate_derives_mantle(): rec = _recommend(_workload(requirements={"data_residency": "in_region_required"})) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py index 6a565409f..deea89a1e 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py @@ -429,6 +429,356 @@ def test_in_region_required_with_guardrails_stays_on_runtime_and_blocks(): assert "cris_residency_unresolved" in _codes(rec["blocks"]) +# --- Invocation logging vs. a Mantle selector (model-invocation-logging.html, +# read 2026-11-08: logging is captured only on bedrock-runtime) -------------- + +def _option(rec, api_path): + return next(o for o in rec["decision_options"] if o["api_path"] == api_path) + + +def test_logging_plus_explicit_selector_is_a_decision_between_mantle_and_runtime_openai_api(): + out = _full_recommend(_workload(requirements={ + "governance": ["invocation_logging"], "mantle_selector": "projects", + })) + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == "decision_required" + assert rec["mantle_selector"] is None + assert {o["api_path"] for o in rec["decision_options"]} == { + "mantle_openai_responses", "runtime_openai_responses", + } + assert _option(rec, "mantle_openai_responses")["mantle_selector"] == "projects" + assert _option(rec, "runtime_openai_responses")["mantle_selector"] is None + assert "invocation logging" in rec["blocks"][0]["message"] + assert "CloudWatch" not in rec["blocks"][0]["message"] + + +def test_logging_plus_derived_in_region_is_a_decision_carrying_the_derived_selector(): + out = _full_recommend(_workload(requirements={ + "governance": ["invocation_logging"], "data_residency": "in_region_required", + })) + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == "decision_required" + assert "model_path_decision_required" in _codes(rec["blocks"]) + assert {o["api_path"] for o in rec["decision_options"]} == { + "mantle_openai_responses", "runtime_openai_responses", + } + assert _option(rec, "mantle_openai_responses")["mantle_selector"] == "in_region_processing" + assert "in_region_processing" in rec["blocks"][0]["message"] + assert "invocation logging" in rec["blocks"][0]["message"] + + +# --- CloudWatch never constrains the endpoint (monitoring-mantle-metrics.html, +# read 2026-11-08: Mantle publishes AWS/BedrockMantle metrics) --------------- + +def test_cloudwatch_plus_selector_lands_on_mantle_with_the_selector(): + rec = _recommend(_workload(requirements={ + "governance": ["cloudwatch"], "mantle_selector": "projects", + })) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "mantle_openai_responses" + assert rec["mantle_selector"] == "projects" + + +def test_cloudwatch_plus_in_region_derives_the_mantle_selector(): + rec = _recommend(_workload(requirements={ + "governance": ["cloudwatch"], "data_residency": "in_region_required", + })) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "mantle_openai_responses" + assert rec["mantle_selector"] == "in_region_processing" + + +def test_cloudwatch_plus_logging_plus_selector_is_still_a_decision(): + rec = _recommend(_workload(requirements={ + "governance": ["cloudwatch", "invocation_logging"], "mantle_selector": "projects", + })) + assert rec["decision_status"] == "decision_required" + assert "model_path_decision_required" in _codes(rec["blocks"]) + + +# --- Guardrails follow the model card's API scope --------------------------- +# +# GPT-6 Sol / GPT-6 Luna list Guardrails on Responses, Chat Completions, Invoke +# and Converse; GPT-6.1 Sol on Chat Completions, Invoke and Converse (not +# Responses); every other GPT card lists Converse only (cards read 2026-11-08). + +@pytest.mark.parametrize(("source_id", "surface", "path"), [ + ("gpt-6-sol", "responses", "runtime_openai_responses"), + ("gpt-6-sol", "chat_completions", "runtime_openai_chat"), + ("gpt-6-luna", "responses", "runtime_openai_responses"), + ("gpt-6.1-sol", "chat_completions", "runtime_openai_chat"), +]) +def test_guardrails_listed_on_the_openai_api_keep_the_source_api_and_model( + source_id, surface, path +): + out = _full_recommend( + _workload( + source={"model_ids": [source_id], "api_surface": surface}, + requirements={ + "api_continuity": "required", "governance": ["guardrails"], + "data_residency": "global_allowed", + }, + ), + region="us-east-1", + ) + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == path + assert rec["primary_model"] == f"openai.{source_id}" + assert rec["invocation_model_id"] == f"global.openai.{source_id}" + assert rec["mantle_selector"] is None + assert "model_path_decision_required" not in _codes(rec["blocks"]) + assert OPENAI_CATALOG["models"][oai._SOURCE_MODEL_KEYS[source_id]]["paths"][path][ + "guardrails_supported"] is True + + +def test_gpt61_sol_responses_with_guardrails_and_continuity_is_a_decision(): + # The GPT-6.1 Sol card scopes Guardrails away from Responses. + rec = _recommend( + _workload( + source={"model_ids": ["gpt-6.1-sol"]}, + requirements={"api_continuity": "required", "governance": ["guardrails"], + "data_residency": "global_allowed"}, + ), + region="us-east-1", + ) + assert rec["decision_status"] == "decision_required" + assert "model_path_decision_required" in _codes(rec["blocks"]) + assert {o["api_path"] for o in rec["decision_options"]} == { + "runtime_openai_responses", "runtime_converse", + } + assert "Converse" in _option(rec, "runtime_openai_responses")["reason"] + assert _option(rec, "runtime_converse")["model"] == "openai.gpt-6.1-sol" + + +def test_guardrails_without_continuity_keep_converse_only_models_on_converse(): + rec = _recommend(_workload(requirements={"governance": ["guardrails"]})) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "runtime_converse" + assert rec["primary_model"] == "openai.gpt-5.4" + + +def test_guardrails_on_the_openai_api_plus_selector_offers_the_runtime_openai_api(): + rec = _recommend( + _workload( + source={"model_ids": ["gpt-6-sol"]}, + requirements={"governance": ["guardrails"], "mantle_selector": "server_side_tools"}, + ), + region="us-east-1", + ) + assert rec["decision_status"] == "decision_required" + assert {o["api_path"] for o in rec["decision_options"]} == { + "mantle_openai_responses", "runtime_openai_responses", + } + assert "Guardrails are listed on this API" in _option(rec, "runtime_openai_responses")["reason"] + assert _option(rec, "mantle_openai_responses")["mantle_selector"] == "server_side_tools" + + +# --- An explicit bedrock-runtime preference never coexists with a selector ---- + +def test_preferred_runtime_path_plus_explicit_selector_is_a_preference_conflict(): + out = _full_recommend(_workload(requirements={ + "preferred_api_path": "runtime_openai_responses", + "mantle_selector": "background_inference", + })) + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == "decision_required" + assert "preferred_api_path_conflict" in _codes(rec["blocks"]) + assert {o["api_path"] for o in rec["decision_options"]} == { + "mantle_openai_responses", "runtime_openai_responses", + } + assert _option(rec, "mantle_openai_responses")["mantle_selector"] == "background_inference" + assert "background_inference" in rec["blocks"][0]["message"] + + +def test_preferred_runtime_path_plus_derived_in_region_is_a_preference_conflict(): + out = _full_recommend(_workload(requirements={ + "preferred_api_path": "runtime_openai_responses", + "data_residency": "in_region_required", + })) + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == "decision_required" + assert "preferred_api_path_conflict" in _codes(rec["blocks"]) + assert _option(rec, "mantle_openai_responses")["mantle_selector"] == "in_region_processing" + assert _option(rec, "runtime_openai_responses")["mantle_selector"] is None + assert "in_region_required" in rec["blocks"][0]["message"] + + +def test_preferred_runtime_path_plus_detected_web_search_is_a_preference_conflict(): + rec = _recommend(_workload( + detected_features=["web_search"], + requirements={"preferred_api_path": "runtime_openai_responses"}, + )) + assert rec["decision_status"] == "decision_required" + assert "preferred_api_path_conflict" in _codes(rec["blocks"]) + assert _option(rec, "mantle_openai_responses")["mantle_selector"] == "web_search" + + +@pytest.mark.parametrize("requirements", [ + {"mantle_selector": "projects"}, + {"mantle_selector": "background_inference"}, + {"data_residency": "in_region_required"}, + {"mantle_selector": "web_search", "governance": ["cloudwatch"]}, +]) +def test_a_recommended_runtime_path_never_carries_a_selector(requirements): + rec = _recommend(_workload(requirements=requirements)) + if rec["decision_status"] == "recommended": + assert rec["api_path"].startswith("mantle_") + assert rec["mantle_selector"] is not None + else: + assert any(o["api_path"].startswith("mantle_") for o in rec["decision_options"]) + + +# --- Background inference is a Mantle Responses API feature +# (inference-responses-api.html, read 2026-11-08) ---------------------------- + +def test_chat_source_with_background_inference_is_a_decision_between_mantle_responses_and_runtime_chat(): + out = _full_recommend(_workload( + source={"api_surface": "chat_completions"}, + requirements={"mantle_selector": "background_inference"}, + )) + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == "decision_required" + assert "selector_api_incompatible" in _codes(rec["blocks"]) + assert _cites_aws_docs(rec["blocks"][0]["message"]) + assert {o["api_path"] for o in rec["decision_options"]} == { + "mantle_openai_responses", "runtime_openai_chat", + } + assert _option(rec, "mantle_openai_responses")["mantle_selector"] == "background_inference" + assert _option(rec, "runtime_openai_chat")["mantle_selector"] is None + + +def test_chat_source_with_background_inference_and_free_continuity_reshapes_to_mantle_responses(): + rec = _recommend(_workload( + source={"api_surface": "chat_completions"}, + requirements={"mantle_selector": "background_inference", + "api_continuity": "not_required"}, + )) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "mantle_openai_responses" + assert rec["mantle_selector"] == "background_inference" + assert "chat_completions_to_responses_required" in _codes(rec["blocks"]) + + +def test_preferred_mantle_chat_with_background_inference_is_a_decision(): + rec = _recommend(_workload( + source={"api_surface": "chat_completions"}, + requirements={"preferred_api_path": "mantle_openai_chat", + "mantle_selector": "background_inference"}, + )) + assert rec["decision_status"] == "decision_required" + assert _codes(rec["blocks"]) & {"preferred_api_path_conflict", "selector_api_incompatible"} + assert _option(rec, "mantle_openai_responses")["mantle_selector"] == "background_inference" + + +# --- GPT-6.1 Sol lists server-side system tools as unsupported on bedrock-mantle +# (model-card-openai-gpt-6-1-sol.html, read 2026-11-08) ----------------------- + +@pytest.mark.parametrize(("surface", "selector", "path"), [ + ("responses", "server_side_tools", "mantle_openai_responses"), + ("chat_completions", "web_search", "mantle_openai_chat"), +]) +def test_gpt61_sol_server_side_tool_selectors_fall_to_astra_and_keep_the_selector( + surface, selector, path +): + requirements = {"mantle_selector": selector} + if surface == "responses": + requirements["api_continuity"] = "required" + out = _full_recommend( + _workload(source={"model_ids": ["gpt-6.1-sol"], "api_surface": surface}, + requirements=requirements), + region="us-east-1", + ) + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == path + assert rec["primary_model"] == "openai.gpt-6-astra" + assert rec["mantle_selector"] == selector + assert "fallback" in rec["rationale"][0] + assert "server-side system tools" in rec["rationale"][0] + + +def test_catalog_walk_skips_mantle_rows_that_deny_server_side_tools_only_for_tool_selectors(): + order = ["openai_gpt_6_1_sol", "openai_gpt_6_astra"] + for selector in ("server_side_tools", "web_search"): + hit, _ = oai._catalog_model_for_path( + OPENAI_CATALOG, "mantle_openai_responses", [], {}, candidate_order=order, + region="us-east-1", selector=selector, + ) + assert hit[0] == "openai_gpt_6_astra" + hit, _ = oai._catalog_model_for_path( + OPENAI_CATALOG, "mantle_openai_responses", [], {}, candidate_order=order, + region="us-east-1", selector="projects", + ) + assert hit[0] == "openai_gpt_6_1_sol" + + +# --- Application inference profiles are Converse-only on bedrock-runtime +# (inference-responses-api.html, read 2026-11-08: Responses/Chat return HTTP 400) -- + +_APP_PROFILE = "arn:aws:bedrock:us-east-2:123456789012:application-inference-profile/app1" + + +def test_application_profile_without_continuity_is_recommended_on_converse_with_the_arn(): + out = _full_recommend(_workload(requirements={ + "data_residency": "geo_required", "cris_geography": "us", + "inference_profile_id": _APP_PROFILE, + })) + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "runtime_converse" + assert rec["primary_model"] == "openai.gpt-5.4" + assert rec["invocation_model_id"] == _APP_PROFILE + + +def test_application_profile_with_continuity_is_a_decision_whose_openai_option_has_no_id(): + out = _full_recommend(_workload(requirements={ + "api_continuity": "required", "data_residency": "geo_required", + "cris_geography": "us", "inference_profile_id": _APP_PROFILE, + })) + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == "decision_required" + assert "application inference profile" in rec["blocks"][0]["message"] + openai_option = _option(rec, "runtime_openai_responses") + assert openai_option["invocation_model_id"] is None + assert "400" in openai_option["reason"] + assert _cites_aws_docs(openai_option["reason"]) + assert _option(rec, "runtime_converse")["invocation_model_id"] == _APP_PROFILE + + +def test_application_profile_with_a_preferred_openai_api_is_a_preference_conflict(): + rec = _recommend(_workload(requirements={ + "preferred_api_path": "runtime_openai_responses", "data_residency": "geo_required", + "cris_geography": "us", "inference_profile_id": _APP_PROFILE, + })) + assert rec["decision_status"] == "decision_required" + assert "preferred_api_path_conflict" in _codes(rec["blocks"]) + assert "application inference profile" in rec["blocks"][0]["message"] + assert all( + o["invocation_model_id"] != _APP_PROFILE + for o in rec["decision_options"] if o["api_path"] != "runtime_converse" + ) + + +def test_system_profile_with_continuity_stays_on_runtime_responses_with_that_id(): + rec = _recommend(_workload(requirements={ + "inference_profile_id": "us.openai.gpt-5.4", "api_continuity": "required", + "data_residency": "geo_required", "cris_geography": "us", + })) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "runtime_openai_responses" + assert rec["invocation_model_id"] == "us.openai.gpt-5.4" + + # --- GPT-6 exact-model routing -------------------------------------------- _GPT6_MODELS = ( @@ -457,7 +807,14 @@ def test_gpt6_supported_region_selects_exact_model_and_validates_schema( rec = out["workloads"]["chat-svc"] assert oai.detect_family(source_id) == "reasoning" assert rec["decision_status"] == "recommended" - assert rec["primary_model"] == target_id + if selector and source_id == "gpt-6.1-sol": + # The GPT-6.1 Sol card lists server-side system tools as unsupported on + # bedrock-mantle (read 2026-11-08): the selector is kept and the walk falls + # to the next same-family model that serves them. + assert rec["primary_model"] == "openai.gpt-6-astra" + assert "fallback" in rec["rationale"][0] + else: + assert rec["primary_model"] == target_id assert rec["api_path"] == path assert rec["mantle_selector"] == selector if critical: @@ -485,11 +842,19 @@ def test_gpt6_governance_selects_exact_runtime_model_and_cris_id( ) assert rec["decision_status"] == "recommended" assert rec["primary_model"] == target_id - assert rec["api_path"] == "runtime_converse" assert rec["invocation_model_id"] == prefix + target_id - delta = _delta_by_code(rec, "openai_sdk_to_converse") - assert "identity is preserved" in delta["description"] - assert "Bedrock-native model" not in delta["description"] + if source_id in {"gpt-6-sol", "gpt-6-luna"}: + # Their cards list Guardrails on the Responses API (read 2026-11-08), so the + # source API stays on bedrock-runtime with the same model and CRIS id. + assert rec["api_path"] == "runtime_openai_responses" + assert "Guardrails are listed on Responses" in rec["rationale"][0] + assert "openai_sdk_to_converse" not in _delta_codes(rec) + else: + # GPT-6 Astra and GPT-6.1 Sol scope Guardrails to Converse for a Responses source. + assert rec["api_path"] == "runtime_converse" + delta = _delta_by_code(rec, "openai_sdk_to_converse") + assert "identity is preserved" in delta["description"] + assert "Bedrock-native model" not in delta["description"] @pytest.mark.parametrize( @@ -511,11 +876,21 @@ def test_gpt6_unsupported_geo_profile_falls_back_without_fabricated_id( _workload(source={"model_ids": [source_id]}, requirements=requirements), region=region, ) - assert rec["decision_status"] == "recommended" - assert rec["invocation_model_id"] != unsupported_id - assert not rec["invocation_model_id"].endswith( - f"openai.{source_id}" - ) + if source_id == "gpt-6-sol": + # Guardrails are on the Responses API for GPT-6 Sol, so the path stays + # runtime_openai_responses; no OpenAI model publishes an EU profile, so the + # engine fails closed instead of fabricating one or switching families silently. + assert rec["decision_status"] == "decision_required" + assert "inference_profile_unverified" in _codes(rec["blocks"]) + assert unsupported_id not in { + option["invocation_model_id"] for option in rec["decision_options"] + } + else: + assert rec["decision_status"] == "recommended" + assert rec["invocation_model_id"] != unsupported_id + assert not rec["invocation_model_id"].endswith( + f"openai.{source_id}" + ) conflict = _recommend( _workload( @@ -556,7 +931,13 @@ def test_gpt6_chat_completions_preserves_chat_surface(source_id, target_id, sele ), region="us-east-1", ) - assert rec["primary_model"] == target_id + if selector and source_id == "gpt-6.1-sol": + # GPT-6.1 Sol lists server-side system tools as unsupported on bedrock-mantle + # (card read 2026-11-08): same-family fallback on the same Chat surface. + assert rec["primary_model"] == "openai.gpt-6-astra" + assert rec["mantle_selector"] == selector + else: + assert rec["primary_model"] == target_id assert rec["api_path"] == path assert "chat_completions_to_responses_required" not in _codes(rec["blocks"]) @@ -757,12 +1138,18 @@ def test_gpt6_chat_findings_preserve_chat_request_shapes(selector, path): assert "previous_response_id" not in state_delta["description"] -@pytest.mark.parametrize(("selector", "openai_path"), [ - (None, "runtime_openai_chat"), - ("server_side_tools", "mantle_openai_chat"), +@pytest.mark.parametrize(("selector", "openai_path", "governance", "other_path"), [ + # multi-model Converse is Converse by definition: continuity vs Converse decision. + (None, "runtime_openai_chat", {"multi_model_converse": True}, "runtime_converse"), + # GPT-6 Sol lists Guardrails on Chat Completions (card read 2026-11-08), so the + # runtime side of the selector-vs-governance decision keeps the Chat API. + ("server_side_tools", "mantle_openai_chat", {"governance": ["guardrails"]}, + "runtime_openai_chat"), ]) -def test_gpt6_chat_conflict_outside_mantle_region_is_schema_valid(selector, openai_path): - requirements = {"api_continuity": "required", "governance": ["guardrails"]} +def test_gpt6_chat_conflict_outside_mantle_region_is_schema_valid( + selector, openai_path, governance, other_path +): + requirements = {"api_continuity": "required", **governance} if selector: requirements["mantle_selector"] = selector workload = _workload( @@ -775,7 +1162,7 @@ def test_gpt6_chat_conflict_outside_mantle_region_is_schema_valid(selector, open assert rec["decision_status"] == "decision_required" assert {option["api_path"] for option in rec["decision_options"]} == { openai_path, - "runtime_converse", + other_path, } option = next( option for option in rec["decision_options"] @@ -1714,6 +2101,68 @@ def test_astra_json_schema_does_not_claim_streaming_support(status, expected): assert "structured_output_streaming_unverified" in _codes(rec["blocks"]) +# The Astra card scopes bedrock-runtime JSON Schema output to non-streaming calls on +# Chat Completions, Responses and Converse alike (model-card-openai-gpt-6-astra.html, +# read 2026-11-08), so the guard covers every runtime API, not only Converse. +@pytest.mark.parametrize("surface,path", [ + ("responses", "runtime_openai_responses"), + ("chat_completions", "runtime_openai_chat"), +]) +@pytest.mark.parametrize("status,expected", [ + ("detected", "decision_required"), ("absent", "recommended"), +]) +def test_astra_runtime_openai_apis_apply_the_non_streaming_json_schema_guard( + surface, path, status, expected +): + out = _full_recommend(_workload( + source={"model_ids": ["gpt-6-astra"], "api_surface": surface}, + requirements={"api_continuity": "required", "data_residency": "global_allowed", + "critical_features": ["structured_output_json"]}, + detected_features=["streaming"], feature_status={"streaming": status}, + ), region="us-east-1") + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == expected + if expected == "decision_required": + block = next(b for b in rec["blocks"] if b["code"] == "structured_output_streaming_unverified") + assert "non-streaming" in block["message"] + assert _cites_aws_docs(block["message"]) + assert _option(rec, path)["model"] == "openai.gpt-6-astra" + else: + assert rec["api_path"] == path + assert rec["primary_model"] == "openai.gpt-6-astra" + + +def test_astra_runtime_responses_structured_output_alone_is_recommended_with_the_text_format_delta(): + rec = _recommend(_workload( + source={"model_ids": ["gpt-6-astra"]}, + requirements={"api_continuity": "required", "data_residency": "global_allowed", + "critical_features": ["structured_output_json"]}, + ), region="us-east-1") + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "runtime_openai_responses" + assert "structured_output_json" in rec["compatibility"]["native"] + assert "structured_output_text_format" in _delta_codes(rec) + assert "structured_output_streaming_unverified" not in _codes(rec["blocks"]) + + +def test_non_streaming_json_schema_guard_is_keyed_on_astra_not_on_the_source(): + # The guard is Astra-specific. A GPT-6.1 Sol source needing both features has no + # GPT-6.1 Sol capability evidence for them, so the walk falls to Astra; any block + # is then about the Astra target, never about GPT-6.1 Sol. + rec = _recommend(_workload( + source={"model_ids": ["gpt-6.1-sol"]}, + requirements={"api_continuity": "required", "data_residency": "global_allowed", + "critical_features": ["structured_output_json"]}, + detected_features=["streaming"], + ), region="us-east-1") + if "structured_output_streaming_unverified" in _codes(rec["blocks"]): + assert _option(rec, "runtime_openai_responses")["model"] == "openai.gpt-6-astra" + assert "GPT-6.1" not in rec["blocks"][0]["message"] + else: + assert rec["primary_model"] != "openai.gpt-6-astra" + + @pytest.mark.parametrize("profile", [ "arn:aws:bedrock:us-east-1:111122223333:inference-profile/us.openai.gpt-6-astra", "arn:aws-cn:bedrock:us-west-2:111122223333:inference-profile/us.openai.gpt-6-astra", diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py index 0f3ba68e5..8c8c4c991 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_verify_model_path.py @@ -651,6 +651,9 @@ def test_changed_in_region_allowance_invalidates_same_arn_verification(tmp_path) # Mantle OpenAI paths are probed only when a mantle_selector put the workload there. MANTLE_OPENAI = {"api_continuity": "required", "mantle_selector": "background_inference"} +# Background inference is a Responses API feature on bedrock-mantle, so a Chat +# Completions source needs a Chat-compatible selector to land on mantle_openai_chat. +MANTLE_OPENAI_CHAT = {"api_continuity": "required", "mantle_selector": "projects"} def test_mantle_responses_probe_calls_responses_create_with_exact_id(): @@ -703,7 +706,7 @@ def test_default_openai_recommendation_probes_runtime_responses_not_mantle(): def test_mantle_chat_probe_calls_chat_completions_with_exact_id(): client = FakeOpenAIClient() result = _verify_openai( - MANTLE_OPENAI, + MANTLE_OPENAI_CHAT, source={"model_ids": ["gpt-6-sol"], "api_surface": "chat_completions"}, region="us-east-1", openai_chat_client_factory=lambda region: client, @@ -734,7 +737,7 @@ def test_mantle_chat_failure_does_not_substitute_model(): completions = FakeCompletions(error=RuntimeError("access denied")) client = FakeOpenAIClient(chat=FakeChat(completions)) result = _verify_openai( - MANTLE_OPENAI, + MANTLE_OPENAI_CHAT, source={"model_ids": ["gpt-6-sol"], "api_surface": "chat_completions"}, region="us-east-1", openai_chat_client_factory=lambda region: client, @@ -892,6 +895,32 @@ def test_runtime_chat_failure_does_not_substitute_model(): assert [call["model"] for call in completions.calls] == ["us.openai.gpt-6-sol"] +@pytest.mark.parametrize("path,factory_kwarg", [ + ("runtime_openai_responses", "runtime_openai_responses_client_factory"), + ("runtime_openai_chat", "runtime_openai_chat_client_factory"), +]) +def test_application_profile_on_a_runtime_openai_api_needs_resolution_without_a_client( + path, factory_kwarg +): + # bedrock-runtime Responses/Chat Completions reject application inference profiles + # with HTTP 400 (inference-responses-api.html, read 2026-11-08): the verifier says + # so before building any client, instead of probing an id the API cannot accept. + recommendation = _runtime_recommendation("openai", path, "openai.gpt-5.4", APPLICATION_ARN) + result = verify_model_path.verify_recommendation( + recommendation, now=NOW, + **{factory_kwarg: lambda region: pytest.fail("client must not be built")}, + ) + jsonschema.validate(result, VERIFICATION_SCHEMA, format_checker=jsonschema.FormatChecker()) + verification = result["workloads"]["openai-svc"] + + assert verification["status"] == "needs_resolution" + assert verification["error"]["type"] == "ApplicationProfileUnsupportedOnApi" + assert "inference-responses-api" in verification["error"]["message"] + assert "Converse" in verification["error"]["message"] + assert verification["invocation_model_id"] == APPLICATION_ARN + assert verification["response_model_id"] is None + + def test_runtime_messages_probe_calls_messages_create_with_exact_cris_id(): messages = FakeMessages(response={"model": "global.anthropic.claude-sonnet-5"}) client = FakeMantleClient(messages) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/verify_model_path.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/verify_model_path.py index eada4fd1c..3c4b92606 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/verify_model_path.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/verify_model_path.py @@ -239,6 +239,21 @@ def verify_workload( return result path = result["api_path"] + if (":application-inference-profile/" in model_id + and path in {"runtime_openai_responses", "runtime_openai_chat"}): + # No client is built: the API rejects this profile before any probe could run. + result["status"] = "needs_resolution" + result["error"] = { + "type": "ApplicationProfileUnsupportedOnApi", + "message": ( + "Application inference profiles are rejected with HTTP 400 on the " + "bedrock-runtime Responses and Chat Completions APIs " + "(https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, " + "read 2026-11-08); select Converse for this profile or a system / Geo / Global " + "inference profile for the OpenAI API." + ), + } + return result try: if (":application-inference-profile/" in model_id and result["path_model_id"] == "anthropic.claude-opus-5-5"): From 948612481a53f1ddfb79cd8c1261318b53854c6a Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 19:27:28 -0800 Subject: [PATCH 35/41] docs(aws-startup-advisor): agent-advisor injection carries mantle_selector/mantle_requirement; Guardrails scope per model card; logging-vs-selector decisions documented - migration-plan.md: units[] template in migration-plan-injection.json carries mantle_selector and mantle_requirement; Step 3.5 copies both from the saved injection, never from conversation context; comparator accepts the us-gov. CRIS prefix. handoff-migration.md Step 1 states the injection mirrors the per-unit fields. - model-recommend.md: Guardrails select runtime_converse only where the source model's card lists Guardrails as Converse-only (GPT-6 Sol/Luna keep Responses/Chat, GPT-6.1 Sol keeps Chat); invocation logging + any selector is a decision; CloudWatch never changes the path; application inference profiles require Converse; background inference requires Mantle Responses; GPT-6.1 Sol server-side tools fall to the next same-family model; runtime preferred_api_path + selector is a preferred_api_path_conflict; Astra non-streaming JSON Schema guard; Anthropic conflict names. - model-selection.md, gcp design-ai.md, shared ai-openai-to-bedrock.md (+ two vendored copies): Guardrails API scope per model card, read 2026-11-08. - test_runtime_default_docs.py: injection template + round-trip tests, us-gov. comparator row. Internal label removed from a test comment. Verification (worktree, this commit): - skills/agent-advisor/scripts: uv run --group dev pytest -q -> 789 passed - skills/llm-to-bedrock/scripts: uv run --group dev pytest -q -> 655 passed (no files changed there) - python3 tools/sync-vendored.py --check -> PASS (65 files) - markdownlint-cli2 on the 8 touched .md files -> 0 errors (the mise lint:md glob also walks the untracked skills/llm-to-bedrock/scripts/.venv) - mise run lint:model-staleness, lint:model-catalog-drift, lint:startup-model-pricing, lint:artifacts, lint:vendored-parity, lint:skill-refs, lint:consent-disclosure -> all pass - plugin tests: 1495 passed, 222 subtests; 1 pre-existing failure tests/test_model_id_lint.py::test_real_plugin_passes (untracked .venv); mise run lint:model-id hits the same untracked .venv - mise run test:telemetry -> 311 passed; test:heroku-source-review -> 44 pass - repo root: python3 tools/validate.py -> All validations passed; git diff --check clean --- .../decision-refs/model-selection.md | 8 +-- .../references/handoff/handoff-migration.md | 3 ++ .../phases/migration-plan/migration-plan.md | 11 ++-- .../phases/model-recommend/model-recommend.md | 53 +++++++++++++++++-- .../test_openai_model_recommendation.py | 2 +- .../scripts/test_runtime_default_docs.py | 38 +++++++++++++ .../vendored/ai/ai-openai-to-bedrock.md | 5 +- .../references/phases/design/design-ai.md | 12 ++++- .../vendored/ai/ai-openai-to-bedrock.md | 5 +- .../skills/shared/ai/ai-openai-to-bedrock.md | 5 +- 10 files changed, 126 insertions(+), 16 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/decision-refs/model-selection.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/decision-refs/model-selection.md index 5542a6365..a4cd2af9b 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/decision-refs/model-selection.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/decision-refs/model-selection.md @@ -34,9 +34,11 @@ authoritative (a table here would drift). The engine applies these provider-neut `bedrock-runtime` by default; a `mantle_*` path only with a recorded `mantle_selector` from that file; API continuity alone never selects Mantle. - **Conflicts are not precedence.** When two hard requirements cannot share one path (e.g. a - verified Mantle selector plus Converse-only governance), emit `decision_required` with both - tradeoff options and rerun after the user resolves the requirement — never silently let one - side win. + verified Mantle selector plus Converse-only governance such as Guardrails on a model whose + card lists them as Converse-only, or plus invocation logging, which only `bedrock-runtime` + captures), emit `decision_required` with both tradeoff options and rerun after the user + resolves the requirement — never silently let one side win. CloudWatch metrics exist on both + endpoints (`AWS/Bedrock`, `AWS/BedrockMantle`) and never create a conflict. - **A dated snapshot is not a permanent claim.** Catalog data is point-in-time — e.g. Sonnet 4.6 was runtime-only in the `2026-06-24` snapshot, and six days later Sonnet 5 launched with full Mantle Messages support, changing the default recommendation. Every recommendation diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/handoff/handoff-migration.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/handoff/handoff-migration.md index 8dcd05bad..8f4401d7f 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/handoff/handoff-migration.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/handoff/handoff-migration.md @@ -29,6 +29,9 @@ with `evidence` = the `file:line construct` behind the `detected` feature status module derived `web_search` from a detected Web Search tool; `"catalog"` with `evidence` = the catalog file and row for `mantle_only_model_or_api`. A unit on any `runtime_*` path carries no `mantle_requirement`; downstream Design writes the `endpoint_assumption` sentence for it. +Migration Plan Step 2 writes the same per-unit `mantle_selector` and `mantle_requirement` into +`migration-plan-injection.json` `units[]`, so the downstream engine reads them from the saved +artifact rather than from conversation context. The summary also carries this translation table so that `gcp-to-aws` Design and `llm-to-bedrock` write `migration_path` / `required_client_api` / `mantle_selector` / diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/migration-plan/migration-plan.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/migration-plan/migration-plan.md index 56a939005..04bc81452 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/migration-plan/migration-plan.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/migration-plan/migration-plan.md @@ -244,6 +244,8 @@ Write the injection context to `$RUN_DIR/migration-plan-injection.json`: "model": "", "api_path": "", "invocation_model_id": "", + "mantle_selector": "", + "mantle_requirement": "", "source": "", "services": "", "evidence": "" @@ -365,12 +367,14 @@ vocabulary of `references/vendored/ai/bedrock-endpoint-policy.md` on the matched | `runtime_openai_chat` | `runtime_openai_cris` | `chat_completions` | same | | `runtime_converse` on a proprietary GPT model (`openai.gpt-6*`, `openai.gpt-5.6-*`, `openai.gpt-5.5`, `openai.gpt-5.4`) | `runtime_openai_cris` | `converse` | same; `runtime_converse` on a Claude or open-weight model keeps the exact comparison | | `runtime_messages` | `runtime_messages` | `messages` | `aws_model_id` = advisor `invocation_model_id` | -| `mantle_openai_responses` / `mantle_openai_chat` / `mantle_messages` | the same path name | `responses` / `chat_completions` / `messages` | `bedrock_models[].mantle_selector` and `bedrock_models[].mantle_requirement` (`handoff-migration.md` Step 1 shape) copied verbatim from the advisor contract; an advisor Mantle contract whose `mantle_selector` is null is a mismatch (only a legacy contract written before the field existed keeps the exact model/path comparison) | +| `mantle_openai_responses` / `mantle_openai_chat` / `mantle_messages` | the same path name | `responses` / `chat_completions` / `messages` | `bedrock_models[].mantle_selector` and `bedrock_models[].mantle_requirement` (`handoff-migration.md` Step 1 shape) copied verbatim from `units[].mantle_selector` / `units[].mantle_requirement` in `migration-plan-injection.json` (the saved handoff), never from live conversation context; an injection unit on a `mantle_*` path whose `mantle_selector` is null is a mismatch (only a legacy contract written before the field existed keeps the exact model/path comparison) | Every `runtime_*` row carries the `endpoint_assumption` sentence of `references/vendored/ai/schema-design-aws-ai.md` with the advisor `unit_id` as `` and no `mantle_requirement`; a `mantle_*` row's `mantle_requirement` is not part of the comparison -below. +below. GCP reads both Mantle fields from the injection file written in Step 2, so a +`mantle_*` unit whose injection entry lacks `mantle_selector` cannot validate: fix the +handoff (rerun Model Recommend and the handoff) rather than typing the selector into Design. An absent `runtime_api` on a legacy runtime plan means Converse; an explicit Chat/Responses choice is not equivalent. Do not replace a confirmed Global profile with US Geo or strip an ARN. @@ -405,7 +409,8 @@ def matches_advisor_model(advisor, plan_model, migration_path, runtime_api=None, return False profile_id = invocation.rsplit("/", 1)[-1] logical_model = (profile_id.split(".", 1)[1] - if profile_id.startswith(("us.", "in.", "global.")) else profile_id) + if profile_id.startswith(("us.", "in.", "us-gov.", "global.")) + else profile_id) resolved_api = runtime_api or ("converse" if path == "runtime_converse" else None) return (logical_model == model and plan_model == invocation and migration_path == "runtime_openai_cris" diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md index e34ff1353..8640e0aa9 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md @@ -115,10 +115,39 @@ their same-model catalog entry before capability/numeric fallbacks. A `mantle_op selected only with a `mantle_selector`: explicit, or derived from a detected `web_search` tool (the built-in Web Search tool is served on `bedrock-mantle` only) or from `data_residency: in_region_required` on a runtime model that is CRIS-only in the region. Guardrails -or a shared multi-model Converse surface select `runtime_converse` (same model first, Claude tier -as fallback); invocation logging or CloudWatch alone do not change the path. Required API -continuity plus Guardrails is a `decision_required` between two bedrock-runtime options +select `runtime_converse` (same model first, Claude tier as fallback) only when the source +model's card lists Guardrails as Converse-only — GPT-5.6 Sol / Terra / Luna, GPT-5.5, GPT-5.4 and +GPT-6 Astra. GPT-6 Sol and GPT-6 Luna keep `runtime_openai_responses` / `runtime_openai_chat`, and +GPT-6.1 Sol keeps `runtime_openai_chat`, because their cards list Guardrails on those APIs (the +catalog row flag `guardrails_supported`; cards read 2026-11-08: +`https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html`, +`model-card-openai-gpt-6-luna.html`, `model-card-openai-gpt-6-1-sol.html`). A GPT-6.1 Sol +Responses source with Guardrails is a `decision_required` because its card excludes Responses. A +shared multi-model Converse surface always selects `runtime_converse`. Required API continuity +plus Converse-only governance is a `decision_required` between two bedrock-runtime options (`runtime_openai_` and `runtime_converse`); Mantle is never offered without a selector. +Invocation logging pins `bedrock-runtime` (model invocation logging captures only that endpoint, +including its OpenAI-compatible APIs: +`https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html`, read +2026-11-08), so logging plus any `mantle_selector` — explicit, or `in_region_processing` derived +from `in_region_required` — is a `decision_required` between the Mantle option (carrying the +selector) and the runtime option that keeps the source API. CloudWatch alone never changes the +path: `bedrock-runtime` publishes `AWS/Bedrock` metrics and `bedrock-mantle` publishes +`AWS/BedrockMantle` (`https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-mantle-metrics.html`, +read 2026-11-08). An application inference profile ARN in `inference_profile_id` requires +Converse — the bedrock-runtime Responses and Chat Completions APIs reject it with HTTP 400 +(`https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html`, read +2026-11-08) — so with required continuity it is a `decision_required` whose OpenAI-API option +carries no invocation id; system, Geo and Global profiles are unaffected. `background_inference` +is a Responses API feature on `bedrock-mantle` only, so a Chat Completions source with that +selector is a `decision_required` (`selector_api_incompatible`: `mantle_openai_responses` versus +`runtime_openai_chat`) unless `api_continuity: not_required` allows the reshape. The +`server_side_tools` / `web_search` selectors skip Mantle rows whose card lists server-side system +tools as unsupported (`server_side_tools_supported: false` — GPT-6.1 Sol) and fall to the next +same-family model with a fallback rationale; the selector is never dropped. An explicit runtime +`preferred_api_path` plus any selector (explicit or derived) is a `preferred_api_path_conflict` +decision between the Mantle option and the preferred runtime option. A recommended `runtime_*` +record never carries a selector. Fallbacks walk same-family candidates first, newest version first, so a GPT-6 Chat Completions source never falls to GPT-5.4 while Astra or GPT-5.5 is available in the region. Governance selects the exact same GPT model through `runtime_converse` only when the requested Global or @@ -129,7 +158,23 @@ region. Azure OpenAI remains an explicit `provider_module_pending` generic resul For Astra, preserve a compatible explicit API preference before the source default. Keep Astra selected and require a decision when its path, region or feature evidence -cannot satisfy the request; do not silently substitute another model. +cannot satisfy the request; do not silently substitute another model. Astra's JSON Schema output +on `bedrock-runtime` applies to non-streaming calls on Chat Completions, Responses and Converse +(`https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html`, read +2026-11-08), so `structured_output_json` plus `streaming` on any of those runtime paths is a +`structured_output_streaming_unverified` decision. + +The Anthropic module applies the same governance rules to its paths: invocation logging plus a +`mantle_selector` (explicit, or derived `in_region_processing`) is a `decision_required` between +`mantle_messages` and the runtime path that keeps the source API (`runtime_messages` for a +Messages source); CloudWatch alone never changes the path; an explicit +`preferred_api_path: runtime_messages` plus Guardrails or a shared multi-model Converse surface is +a `decision_required` between `runtime_messages` and `runtime_converse`; an explicit runtime +preference plus any selector is a `decision_required` between `mantle_messages` and the preferred +path. The conflict name (`mantle_selector_vs_invocation_logging`, +`in_region_processing_vs_invocation_logging`, `preferred_path_vs_mantle_selector`, +`preferred_path_vs_converse`, `in_region_processing_vs_preferred_api_path`) appears in the +`model_path_decision_required` message. **OpenRouter/LiteLLM-sourced OpenAI models still use the OpenAI module.** Discover records the underlying `provider` (e.g. `openai`) even when the calls transit a gateway — see `discover.md`'s diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py index deea89a1e..8cb8cda11 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py @@ -1950,7 +1950,7 @@ def test_mantle_preference_without_selector_fails_closed_for_every_source(prefer assert not any(o["api_path"].startswith("mantle_") for o in rec["decision_options"]) -# Astra caching evidence (FEAT-001 rows): runtime_openai_responses and +# Astra caching evidence (catalog per-path rows): runtime_openai_responses and # runtime_openai_chat carry prompt_caching; on Mantle only Responses does. The # invariant: a required cache is checked on the API actually selected. @pytest.mark.parametrize("region", ["us-east-1", "us-west-2"]) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_runtime_default_docs.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_runtime_default_docs.py index f3f98c810..f435bfe76 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_runtime_default_docs.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_runtime_default_docs.py @@ -103,6 +103,7 @@ def _comparator(): ("runtime_openai_chat", "chat_completions", "openai.gpt-5.5", "us.openai.gpt-5.5"), ("runtime_converse", "converse", "openai.gpt-5.4", "us.openai.gpt-5.4"), ("runtime_converse", "converse", "openai.gpt-6-astra", "global.openai.gpt-6-astra"), + ("runtime_openai_responses", "responses", "openai.gpt-5.6-terra", "us-gov.openai.gpt-5.6-terra"), ]) def test_comparator_translates_runtime_openai_paths_to_runtime_openai_cris(path, api, model, invocation): matches = _comparator() @@ -148,6 +149,43 @@ def test_comparator_requires_an_equal_non_null_mantle_selector(path, model): assert not matches({**advisor, "mantle_selector": None}, model, path) +def _injection_template(): + section = MIGRATION_PLAN_MD.read_text().split("migration-plan-injection.json`:", 1)[1] + return re.search(r"```json\n(.*?)\n```", section, re.S).group(1) + + +def test_injection_template_carries_selector_and_requirement(): + template = _injection_template() + units = template.split('"units"', 1)[1].split('"platform"', 1)[0] + for key in ('"mantle_selector"', '"mantle_requirement"'): + assert key in units, f"migration-plan-injection.json units[] must carry {key}" + assert units.index('"invocation_model_id"') < units.index('"mantle_selector"') \ + < units.index('"mantle_requirement"') < units.index('"source"') + + +def test_injection_round_trip_preserves_accepted_selector(): + matches = _comparator() + advisor = {"model": "openai.gpt-5.6-sol", "api_path": "mantle_openai_chat", + "invocation_model_id": "openai.gpt-5.6-sol", "mantle_selector": "projects", + "mantle_requirement": {"workload_id": "chat-svc", "selector": "projects", + "source": "user"}} + injection_unit = {k: advisor[k] for k in ("model", "api_path", "invocation_model_id", + "mantle_selector", "mantle_requirement")} + gcp_row = {"aws_model_id": injection_unit["model"], + "migration_path": injection_unit["api_path"], + "required_client_api": "chat_completions", + "mantle_selector": injection_unit["mantle_selector"], + "mantle_requirement": injection_unit["mantle_requirement"]} + assert matches(advisor, gcp_row["aws_model_id"], gcp_row["migration_path"], None, + gcp_row["mantle_selector"]) is True + assert gcp_row["mantle_requirement"] == advisor["mantle_requirement"] + # a legacy injection unit without the selector yields a GCP row without one + legacy_unit = {k: advisor[k] for k in ("model", "api_path", "invocation_model_id")} + assert matches(advisor, legacy_unit["model"], legacy_unit["api_path"], None, None) is False + assert matches(advisor, gcp_row["aws_model_id"], gcp_row["migration_path"], None, + "web_search") is False + + def test_gcp_design_ai_todo_is_resolved(): assert "TODO(PR 3" not in GCP_DESIGN_AI_MD.read_text() diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index b8a8b69c0..5dc0c6375 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -408,7 +408,10 @@ logs, Guardrails and application inference profiles (both Converse API only), an prompt caching (both Responses API only) — per the GPT-5.4 / GPT-5.5 model cards read 2026-11-06. Knowledge Bases and intelligent prompt routing still require a Bedrock-native model. The same-model `bedrock-runtime` path is the default; the Bedrock-native path is the most feature-complete; the mantle GPT path is reserved for a -recorded `mantle_selector`. +recorded `mantle_selector`. Guardrails API scope is per model card (read 2026-11-08): GPT-6 Sol and +GPT-6 Luna list Guardrails on Responses, Chat Completions, InvokeModel and Converse; GPT-6.1 Sol on +Chat Completions, InvokeModel and Converse (not Responses); GPT-5.6 Sol/Terra/Luna, GPT-5.5, GPT-5.4 +and GPT-6 Astra on Converse only — the catalog row flag `guardrails_supported` carries this. ### High-spend tiering diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index ab0a7eeb4..160f0ba14 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -467,8 +467,16 @@ Profiles (Converse-only features that also require Invoke-adjacent plumbing on s must verify against Astra's model card per-feature, not assume parity with Sol/Luna/6.1 Sol. **When Guardrails is the driving need and the design keeps `runtime_openai_cris` (endpoint switch, not model switch), it MUST also record `bedrock_models[].required_client_api` so the -generated client uses a Guardrails-capable surface — `converse` for Astra and 6.1 Terra; `converse` / -`chat_completions` / `invoke` (never `responses`) for 6.1 Sol and the other GPT-6/5.6 CRIS models except Astra and 6.1 Terra. Do not leave the client on `responses` when Guardrails is required.** For a +generated client uses a Guardrails-capable surface — `converse` for GPT-6 Astra, GPT-5.6 Sol / +Terra / Luna, GPT-5.5, GPT-5.4 and GPT-6.1 Terra (their cards list Guardrails as Converse-only); +`responses` / `chat_completions` / `invoke` / `converse` for GPT-6 Sol and GPT-6 Luna; +`chat_completions` / `invoke` / `converse` (never `responses`) for GPT-6.1 Sol — model cards read +2026-11-08 (`model-card-openai-gpt-6-sol.html`, `model-card-openai-gpt-6-luna.html`, +`model-card-openai-gpt-6-1-sol.html`, `model-card-openai-gpt-6-astra.html`, +`model-card-openai-gpt-56-sol.html`, `model-card-openai-gpt-56-terra.html`, +`model-card-openai-gpt-56-luna.html`, `model-card-openai-gpt-55.html`, `model-card-openai-gpt-54.html` +under `https://docs.aws.amazon.com/bedrock/latest/userguide/`). Do not leave the client on +`responses` when Guardrails is required on a model whose card excludes Responses.** For a region-only CRIS move with no Guardrails or application-inference-profile need, the default `required_client_api` is the API the source app already calls (`responses` or `chat_completions`) — never an implicit Converse. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index b8a8b69c0..5dc0c6375 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -408,7 +408,10 @@ logs, Guardrails and application inference profiles (both Converse API only), an prompt caching (both Responses API only) — per the GPT-5.4 / GPT-5.5 model cards read 2026-11-06. Knowledge Bases and intelligent prompt routing still require a Bedrock-native model. The same-model `bedrock-runtime` path is the default; the Bedrock-native path is the most feature-complete; the mantle GPT path is reserved for a -recorded `mantle_selector`. +recorded `mantle_selector`. Guardrails API scope is per model card (read 2026-11-08): GPT-6 Sol and +GPT-6 Luna list Guardrails on Responses, Chat Completions, InvokeModel and Converse; GPT-6.1 Sol on +Chat Completions, InvokeModel and Converse (not Responses); GPT-5.6 Sol/Terra/Luna, GPT-5.5, GPT-5.4 +and GPT-6 Astra on Converse only — the catalog row flag `guardrails_supported` carries this. ### High-spend tiering diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md index b8a8b69c0..5dc0c6375 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md @@ -408,7 +408,10 @@ logs, Guardrails and application inference profiles (both Converse API only), an prompt caching (both Responses API only) — per the GPT-5.4 / GPT-5.5 model cards read 2026-11-06. Knowledge Bases and intelligent prompt routing still require a Bedrock-native model. The same-model `bedrock-runtime` path is the default; the Bedrock-native path is the most feature-complete; the mantle GPT path is reserved for a -recorded `mantle_selector`. +recorded `mantle_selector`. Guardrails API scope is per model card (read 2026-11-08): GPT-6 Sol and +GPT-6 Luna list Guardrails on Responses, Chat Completions, InvokeModel and Converse; GPT-6.1 Sol on +Chat Completions, InvokeModel and Converse (not Responses); GPT-5.6 Sol/Terra/Luna, GPT-5.5, GPT-5.4 +and GPT-6 Astra on Converse only — the catalog row flag `guardrails_supported` carries this. ### High-spend tiering From da15cad486e85d1809647dd14d77594607f40ff0 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Sun, 8 Nov 2026 20:34:37 -0800 Subject: [PATCH 36/41] fix(aws-startup-advisor): agent-advisor runtime_converse strengths list the us-gov. CRIS prefix for GPT-5.6 Terra/Luna The catalog's inference_profiles maps for GPT-5.6 Terra and Luna carry us-gov. (us-gov-east-1, us-gov-west-1) per their model cards read 2026-11-08, but the runtime_converse strengths sentence still said "us./in./global." and cited the 2026-11-07 read. Align the prose with the data (canonical skills/shared catalog, vendored agent-advisor copy via tools/sync-vendored.py) and the test that locks that sentence. Integration verification on this tree (plugins/aws-startup-advisor): - skills/agent-advisor/scripts: uv run --group dev pytest -q -> 789 passed - skills/llm-to-bedrock/scripts: uv run --group dev pytest -q -> 655 passed (no llm-to-bedrock files changed) - python3 tools/sync-vendored.py --check -> PASS (65 files) - mise run ci: lint:md aborts the aggregate because its **/*.md glob walks the gitignored skills/llm-to-bedrock/scripts/.venv (openai package api.md MD033); markdownlint-cli2 on the 822 tracked .md files -> 0 errors. Tasks run individually: lint:model-id fails only on the same .venv (anthropic/_constants.py retired Opus 4.1 id); lint:model-staleness ok; lint:model-catalog-drift OK (3 non-strict warnings owned by #566/#567/#581); lint:startup-model-pricing ok; lint:artifacts PASS 101; lint:vendored-parity PASS; lint:skill-refs PASS 1544; lint:consent-disclosure OK; test -> 1495 passed + 222 subtests, 1 pre-existing failure tests/test_model_id_lint.py::test_real_plugin_passes (same .venv); test:telemetry 311 passed; test:heroku-source-review all pass. - tests/test_gpt54_55_cris_prefix_scope.py, test_model_catalog_drift.py, test_runtime_default_review_round1.py -> 29 passed - repo root: python3 tools/validate.py -> All validations passed; git diff --check clean; no internal task labels in tracked files - not run: mise run build (gitleaks fails on pre-existing upstream f8d09fd1) --- .../references/vendored/ai/models/openai-bedrock.json | 2 +- .../agent-advisor/scripts/test_openai_model_recommendation.py | 4 ++-- .../skills/shared/ai/models/openai-bedrock.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json index 646478d7a..bae98f09f 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json @@ -51,7 +51,7 @@ "strengths": [ "Bedrock-native multi-model request surface", "Supports Bedrock Guardrails (Converse API only) and model invocation logging — both bedrock-runtime-only; CloudWatch metrics exist on both endpoints (AWS/Bedrock on runtime, AWS/BedrockMantle on Mantle; https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-mantle-metrics.html, read 2026-11-07)", - "This is a SAME-MODEL governance path via CRIS. GPT-5.6 Terra and Luna publish us./in./global. ids; us./global. only for GPT-5.6 Sol, GPT-5.5 and GPT-5.4 (their model cards list no in. profile; cards read 2026-11-07). The base URL for the OpenAI-compatible APIs is bedrock-runtime.{region}.amazonaws.com/openai/v1. The model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" + "This is a SAME-MODEL governance path via CRIS. GPT-5.6 Terra and Luna publish us./in./us-gov./global. ids; us./global. only for GPT-5.6 Sol, GPT-5.5 and GPT-5.4 (their model cards list no in. or us-gov. profile; cards read 2026-11-08). The base URL for the OpenAI-compatible APIs is bedrock-runtime.{region}.amazonaws.com/openai/v1. The model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" ], "constraints": [ "GPT-5.5 / GPT-5.4 on runtime are CRIS-only (us. and global. ids; no in-region form) per their model cards (read 2026-11-06); the cards publish In-Region (1.10x) pricing only, so do not assume Global CRIS price parity for them", diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py index 8cb8cda11..7722de8cb 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py @@ -209,8 +209,8 @@ def test_runtime_converse_path_text_does_not_grant_in_profile_to_gpt54_gpt55(): strengths = "\n".join(path["strengths"]) constraints = "\n".join(path["constraints"]) assert "For GPT-5.6, GPT-5.5 and GPT-5.4" not in strengths - # in. exists for GPT-5.6 Terra/Luna only; Sol, 5.5 and 5.4 are us./global. - assert "GPT-5.6 Terra and Luna publish us./in./global. ids" in strengths + # in. and us-gov. exist for GPT-5.6 Terra/Luna only; Sol, 5.5 and 5.4 are us./global. + assert "GPT-5.6 Terra and Luna publish us./in./us-gov./global. ids" in strengths assert "us./global. only for GPT-5.6 Sol, GPT-5.5 and GPT-5.4" in strengths assert "For GPT-5.6, runtime does not support server-side tool use" in constraints # The GPT-5.6 cards (read 2026-11-08) document non-streaming JSON Schema on diff --git a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json index 646478d7a..bae98f09f 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json @@ -51,7 +51,7 @@ "strengths": [ "Bedrock-native multi-model request surface", "Supports Bedrock Guardrails (Converse API only) and model invocation logging — both bedrock-runtime-only; CloudWatch metrics exist on both endpoints (AWS/Bedrock on runtime, AWS/BedrockMantle on Mantle; https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-mantle-metrics.html, read 2026-11-07)", - "This is a SAME-MODEL governance path via CRIS. GPT-5.6 Terra and Luna publish us./in./global. ids; us./global. only for GPT-5.6 Sol, GPT-5.5 and GPT-5.4 (their model cards list no in. profile; cards read 2026-11-07). The base URL for the OpenAI-compatible APIs is bedrock-runtime.{region}.amazonaws.com/openai/v1. The model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" + "This is a SAME-MODEL governance path via CRIS. GPT-5.6 Terra and Luna publish us./in./us-gov./global. ids; us./global. only for GPT-5.6 Sol, GPT-5.5 and GPT-5.4 (their model cards list no in. or us-gov. profile; cards read 2026-11-08). The base URL for the OpenAI-compatible APIs is bedrock-runtime.{region}.amazonaws.com/openai/v1. The model cards recommend runtime for new applications, and for GPT-5.6 Global CRIS is priced at OpenAI's standard rate (cost parity)" ], "constraints": [ "GPT-5.5 / GPT-5.4 on runtime are CRIS-only (us. and global. ids; no in-region form) per their model cards (read 2026-11-06); the cards publish In-Region (1.10x) pricing only, so do not assume Global CRIS price parity for them", From 0f4c56e2beb6431567437156cf8dcc93bf4fa16e Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 9 Nov 2026 10:40:37 -0800 Subject: [PATCH 37/41] chore(aws-startup-advisor): re-sync agent-advisor vendored copies after rebasing onto PR 585 Rebased fix/agent-advisor-runtime-default from e6795f32 onto the new PR 585 tip 4f861123 (backup ref backup/pr588-pre-rebase-91827f91); the ten commits replayed without conflicts. tools/sync-vendored.py re-synced skills/agent-advisor/references/vendored/ai/bedrock-endpoint-policy.md (F13 on #585). sync-vendored --check: 65 files OK. --- .../vendored/ai/bedrock-endpoint-policy.md | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-endpoint-policy.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-endpoint-policy.md index 6110b3298..4f1771cfa 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-endpoint-policy.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/bedrock-endpoint-policy.md @@ -4,7 +4,8 @@ > Vendored into each consuming skill as `references/vendored/ai/bedrock-endpoint-policy.md` > and kept byte-identical by `tools/sync-vendored.py`; edit HERE, then run the sync. > Every statement below comes from the public Amazon Bedrock User Guide pages listed in -> § Sources (read 2026-11-07). Re-read those pages before changing a row. +> § Sources (read 2026-11-07; the GPT-5.4 and GPT-5.5 model cards read 2026-11-08). +> Re-read those pages before changing a row. ## The rule @@ -28,8 +29,9 @@ Anthropic Messages source keeps its SDK on `bedrock-runtime` by changing the bas credential and the model ID. Choosing Mantle because "the app already speaks Responses" is not a selector. -**Pricing never selects an endpoint.** Per-token pricing for the same model is identical on both -endpoints; choose by API and capability need. +**Pricing never selects an endpoint.** Choose by API and capability need. Read the per-token +rate from that model's card for the inference option the endpoint serves. A card with no Geo or +Global row has not published that price; do not use the In-Region rate in its place. ## `mantle_selector` — the recorded reason @@ -80,10 +82,10 @@ The seven values are learned in three different ways; no phase may guess one it client-side `"type": "function"` tool merely named `web_search`. - `server_side_tools` — positive: a tools entry of `"type": "mcp"` (`server_url` / `server_label`), `code_interpreter`, `file_search`, `computer_use_preview`, - `image_generation` used as a tool type, or an Anthropic `code_execution_*` / `text_editor_*` - / `computer_*` / `bash_*` tool type — any provider-executed tool other than web search. Not a - signal: `"type": "function"` tools, Gemini `function_declarations=`, `@tool` decorators, - `tool_choice` — client-side function tools never count. + `image_generation` used as a tool type, or an Anthropic `code_execution_*` tool type — any + provider-executed tool other than web search. Not a signal: `"type": "function"` tools, + Gemini `function_declarations=`, `@tool` decorators, `tool_choice` — client-side function + tools never count; Anthropic `computer_*` (incl. `computer_toolset_*`), `browser_toolset_*`, `text_editor_*`, `bash_*` and `memory_*` are CLIENT-executed tools listed on both `bedrock-runtime` and `bedrock-mantle` by the Sonnet 5 and Opus 5.5 cards (`model-card-anthropic-claude-sonnet-5.html`, `model-card-anthropic-claude-opus-5-5.html`, read 2026-11-09) — a client-tool-only Anthropic app stays `runtime_messages`. - `projects` — positive: an `OpenAI-Project` header (`default_headers={"OpenAI-Project": ...}`), a `project=` kwarg on the OpenAI client, or an `OPENAI_PROJECT_ID`-style environment name read in code; in usage discovery, a @@ -148,6 +150,14 @@ engine (`endpoints.html`). All pages under `https://docs.aws.amazon.com/bedrock/latest/userguide/`, read 2026-11-07: - `endpoints.html` — https://docs.aws.amazon.com/bedrock/latest/userguide/endpoints.html +- `model-card-openai-gpt-54.html` and `model-card-openai-gpt-55.html` (read 2026-11-08) — + commercial In-Region prices only. Runtime lists `us.` and `global.` ids, and those rows + publish no price. +- `model-card-anthropic-claude-sonnet-5.html` and `model-card-anthropic-claude-opus-5-5.html` + (read 2026-11-09) — "Computer use" and "Client tools and toolsets using `bedrock-runtime` and + `bedrock-mantle` endpoints": `computer_20251124`, `computer_toolset_20260801`, + `browser_toolset_20260801`, `bash_20250124`, `text_editor_20250728`, `memory_20250818` are + client-executed on both endpoints. - `models-endpoint-availability.html` - `inference-responses-api.html` - `inference-chat-completions.html` From af7ef41adf66418cc06be21f68aa48b7562e5914 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 9 Nov 2026 10:49:49 -0800 Subject: [PATCH 38/41] fix(aws-startup-advisor): agent-advisor engines validate web_search/background/projects/workspaces selectors against the final provider and API (PR 588 F4) OpenAI module: `_RESPONSES_ONLY_SELECTORS` = {background_inference, web_search}. A Chat Completions source matched by either selector (explicit, detected or critical web_search) is a `selector_api_incompatible` decision between `mantle_openai_responses` (carrying the selector) and `runtime_openai_chat` (web-search.html / inference-responses-api.html, read 2026-11-09); the free continuity reshape to Mantle Responses is unchanged. `_decision_options` never builds a `mantle_openai_chat` option for a Responses-only selector. `workspaces` on an OpenAI SDK workload is a `selector_provider_mismatch` decision whose two options both stay on bedrock-runtime (anthropic-workspace-id vs OpenAI-Project; workspaces.html / projects.html, read 2026-11-09). Anthropic module: `_path_constraints` adds `mantle_selector_vs_anthropic_api` (background_inference: the Messages API has no background mode and this module has no Responses route) and `mantle_selector_provider_mismatch` (projects) with runtime_messages + runtime Converse/InvokeModel as the options; the Mantle option is never offered for those selectors. `_decision_required` emits the dedicated `selector_api_incompatible` / `selector_provider_mismatch` findings. model-recommend.md documents the per-selector validation. Tests: test_openai_model_recommendation.py::{test_gpt55_chat_source_with_web_search_is_a_decision_between_mantle_responses_and_runtime_chat, test_gpt55_chat_with_web_search_and_free_continuity_reshapes_to_mantle_responses, test_chat_source_with_web_search_plus_logging_never_offers_a_mantle_chat_option, test_workspaces_selector_on_an_openai_source_is_a_provider_mismatch_decision}; test_model_recommendation.py::{test_anthropic_messages_with_background_inference_is_a_decision_not_mantle_messages, test_projects_selector_on_an_anthropic_source_is_a_provider_mismatch_decision, test_projects_with_a_preferred_mantle_path_still_never_offers_mantle_messages, test_background_inference_with_native_payload_offers_runtime_messages_or_invoke}. Parametrized selector tests exclude the now-deciding selectors; the GPT-6.1 Sol Chat case uses server_side_tools and a free-continuity web_search row. Validation: skills/agent-advisor/scripts `uv run --group dev pytest -q`: 795 passed (was 789). --- .../phases/model-recommend/model-recommend.md | 21 ++++- .../scripts/anthropic_model_recommendation.py | 47 ++++++++++- .../scripts/openai_model_recommendation.py | 63 +++++++++++--- .../scripts/test_model_recommendation.py | 82 ++++++++++++++++++- .../test_openai_model_recommendation.py | 82 ++++++++++++++++++- 5 files changed, 272 insertions(+), 23 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md index 8640e0aa9..c9fffd327 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md @@ -138,10 +138,23 @@ read 2026-11-08). An application inference profile ARN in `inference_profile_id` Converse — the bedrock-runtime Responses and Chat Completions APIs reject it with HTTP 400 (`https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html`, read 2026-11-08) — so with required continuity it is a `decision_required` whose OpenAI-API option -carries no invocation id; system, Geo and Global profiles are unaffected. `background_inference` -is a Responses API feature on `bedrock-mantle` only, so a Chat Completions source with that -selector is a `decision_required` (`selector_api_incompatible`: `mantle_openai_responses` versus -`runtime_openai_chat`) unless `api_continuity: not_required` allows the reshape. The +carries no invocation id; system, Geo and Global profiles are unaffected. Every selector is +validated against the final provider and API before a path is chosen. `background_inference` and +`web_search` are Responses API features on `bedrock-mantle` only +(`https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html`, +`https://docs.aws.amazon.com/bedrock/latest/userguide/web-search.html`, read 2026-11-09), so a +Chat Completions source matched by either selector is a `decision_required` +(`selector_api_incompatible`: `mantle_openai_responses` carrying the selector versus +`runtime_openai_chat` without it) unless `api_continuity: not_required` allows the reshape; +`mantle_openai_chat` is never recorded or offered for a Responses-only selector. A selector whose +provider does not match the workload is a `selector_provider_mismatch` decision whose options +both stay on `bedrock-runtime`: `workspaces` (`anthropic-workspace-id`, +`https://docs.aws.amazon.com/bedrock/latest/userguide/workspaces.html`) on an OpenAI SDK +workload, and `projects` (`OpenAI-Project`, +`https://docs.aws.amazon.com/bedrock/latest/userguide/projects.html`) on an Anthropic SDK +workload (read 2026-11-09). An Anthropic Messages source with `background_inference` is likewise +a `selector_api_incompatible` decision between `runtime_messages` and the runtime governance path +— the Messages API has no background mode and the Anthropic module has no Responses route. The `server_side_tools` / `web_search` selectors skip Mantle rows whose card lists server-side system tools as unsupported (`server_side_tools_supported: false` — GPT-6.1 Sol) and fall to the next same-family model with a fallback rationale; the selector is never dropped. An explicit runtime diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/anthropic_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/anthropic_model_recommendation.py index 1e10516d8..4b24319fd 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/anthropic_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/anthropic_model_recommendation.py @@ -262,20 +262,34 @@ def _path_constraints(workload): messages_option = "runtime_messages" if (source_messages or messages_required) else runtime_path conflicts = [] option_paths = [] + # A selector is validated against the API this module can route to. Background + # inference is a Responses API feature on bedrock-mantle (inference-responses-api.html, + # read 2026-11-09) and the Messages API has no background mode; `projects` scopes the + # OpenAI-compatible APIs (`OpenAI-Project`, projects.html) while Messages scopes by + # Workspaces (`anthropic-workspace-id`, workspaces.html). Neither can select + # mantle_messages: both options stay on bedrock-runtime (Messages, or the + # Converse/InvokeModel governance path) and drop the selector. + if selector == "background_inference": + conflicts.append("mantle_selector_vs_anthropic_api") + option_paths = ["runtime_messages", runtime_path] + if selector == "projects": + conflicts.append("mantle_selector_provider_mismatch") + option_paths = ["runtime_messages", runtime_path] + routable_selector = selector if selector not in {"background_inference", "projects"} else None if messages_required and (converse_required or native_required): conflicts.append("messages_vs_converse") option_paths = ["runtime_messages", runtime_path] - if selector and (converse_required or native_required): + if routable_selector and (converse_required or native_required): conflicts.append("mantle_selector_vs_runtime") option_paths = ["mantle_messages", runtime_path] - elif selector and logging_required: + elif routable_selector and logging_required: conflicts.append("mantle_selector_vs_invocation_logging") option_paths = ["mantle_messages", messages_option] if preferred: if preferred.startswith("mantle_") and (converse_required or native_required): conflicts.append("preferred_path_vs_runtime") option_paths = option_paths or ["mantle_messages", runtime_path] - if preferred.startswith("runtime_") and selector: + if preferred.startswith("runtime_") and routable_selector: conflicts.append("preferred_path_vs_mantle_selector") option_paths = option_paths or ["mantle_messages", preferred] if preferred == "runtime_messages" and converse_required: @@ -837,6 +851,33 @@ def _decision_required(workload, region, catalog, constraints): "bedrock-runtime option, then rerun Model Recommend.", ) ) + if "mantle_selector_vs_anthropic_api" in constraints["conflicts"]: + blocks.append( + _finding( + "selector_api_incompatible", + "[BLOCKS]", + "Background inference is a Responses API feature on bedrock-mantle " + "(https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, " + "read 2026-11-09); the Anthropic Messages API has no background mode and this " + "module has no OpenAI Responses route.", + "Keep the Messages API on bedrock-runtime and drop background_inference, or " + "move the workload to an OpenAI Responses source as a provider change, then " + "rerun Model Recommend.", + ) + ) + if "mantle_selector_provider_mismatch" in constraints["conflicts"]: + blocks.append( + _finding( + "selector_provider_mismatch", + "[BLOCKS]", + "`projects` scopes the OpenAI-compatible APIs (`OpenAI-Project`, " + "https://docs.aws.amazon.com/bedrock/latest/userguide/projects.html); the " + "Anthropic Messages API scopes by Workspaces (`anthropic-workspace-id`, " + "https://docs.aws.amazon.com/bedrock/latest/userguide/workspaces.html, " + "read 2026-11-09).", + "Record the applicable selector (or none) in Clarify and rerun Model Recommend.", + ) + ) return { "workload_id": workload["workload_id"], "provider_module": "anthropic", diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py index a2d04f01d..bb945ef12 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py @@ -340,6 +340,11 @@ def _hard_profile_requirement(requirements): _SERVER_SIDE_TOOL_SELECTORS = frozenset({"server_side_tools", "web_search"}) +# Selectors served on the bedrock-mantle Responses API only: background inference +# (inference-responses-api.html) and the built-in Web Search tool (web-search.html), +# both read 2026-11-09. Chat Completions has neither, so a Chat source matched by +# one of these is an explicit reshape decision, never a same-API Mantle Chat row. +_RESPONSES_ONLY_SELECTORS = frozenset({"background_inference", "web_search"}) def _catalog_model_for_path(catalog, path, detected_features=None, requirements=None, @@ -1094,6 +1099,9 @@ def _decision_options(catalog, workload, region, selector=None, openai_path=None needed = set(detected) | set(requirements.get("critical_features") or []) if "prompt_caching" in needed: openai_path = "mantle_openai_responses" + if selector in _RESPONSES_ONLY_SELECTORS: + # A Responses-only selector is never offered on a Mantle Chat row. + openai_path = "mantle_openai_responses" else: openai_path = _runtime_openai_path(surface) candidate_order = ( @@ -1379,6 +1387,28 @@ def _preferred_path_conflict(mantle_selector, message): decision_options=_mantle_options(mantle_selector) + _runtime_option(preferred), ) + # --- Provider mismatch: `workspaces` scopes the Anthropic Messages API --- + if selector == "workspaces": + return _unresolved( + "mantle_selector workspaces does not apply to an OpenAI SDK workload.", + [], + _finding( + "selector_provider_mismatch", "[BLOCKS]", + "`workspaces` scopes the Anthropic Messages API (`anthropic-workspace-id`, " + "https://docs.aws.amazon.com/bedrock/latest/userguide/workspaces.html); an " + "OpenAI SDK workload has no Workspaces concept — the OpenAI equivalent is " + "`projects` (`OpenAI-Project`, " + "https://docs.aws.amazon.com/bedrock/latest/userguide/projects.html, " + "read 2026-11-09).", + "Record the applicable selector (or none) in Clarify and rerun Model Recommend.", + ), + openai_path_default, + # Both options stay on bedrock-runtime: the source API, or Converse governance. + decision_options=_decision_options( + catalog, workload, region, None, openai_path=openai_path_default + ), + ) + # --- Hard conflict: a verified Mantle-only need AND bedrock-runtime-only governance --- if selector and runtime_required: drivers = ", ".join(_runtime_only_drivers(requirements)) @@ -1525,21 +1555,32 @@ def _preferred_path_conflict(mantle_selector, message): ) path = preferred - # Background inference is a Responses API feature on bedrock-mantle; a Chat - # Completions Mantle path cannot carry it, so the reshape is offered as a - # decision rather than selected silently. - if selector == "background_inference" and path == "mantle_openai_chat": - return _unresolved( - "mantle_selector background_inference conflicts with the Chat Completions API.", - [], - _finding( - "selector_api_incompatible", "[BLOCKS]", + # Background inference and Web Search are Responses API features on + # bedrock-mantle; a Chat Completions Mantle path cannot carry either, so the + # reshape is offered as a decision rather than selected silently. + if selector in _RESPONSES_ONLY_SELECTORS and path == "mantle_openai_chat": + if selector == "web_search": + message = ( + "Web Search is a server-side tool served on the bedrock-mantle Responses API " + "(https://docs.aws.amazon.com/bedrock/latest/userguide/web-search.html, " + "read 2026-11-09); Chat Completions has no `web_search` tool." + ) + dropped = "Web Search" + else: + message = ( "Background inference is a Responses API feature on bedrock-mantle; Chat " "Completions has no background mode " "(https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html, " - "read 2026-11-08).", + "read 2026-11-08)." + ) + dropped = "background inference" + return _unresolved( + f"mantle_selector {selector} conflicts with the Chat Completions API.", + [], + _finding( + "selector_api_incompatible", "[BLOCKS]", message, "Accept the Mantle Responses reshape, or keep Chat Completions on " - "bedrock-runtime and drop background inference, then rerun Model Recommend.", + f"bedrock-runtime and drop {dropped}, then rerun Model Recommend.", ), path, decision_options=[ diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_model_recommendation.py index 1c4162a50..2111c0d10 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_model_recommendation.py @@ -99,7 +99,14 @@ def test_cost_priority_with_workspaces_selector_uses_clean_mantle_haiku_id(): assert rec["mantle_selector"] == "workspaces" -@pytest.mark.parametrize("selector", anthropic_model_recommendation.MANTLE_SELECTORS) +# background_inference (Responses-only) and projects (OpenAI-only) are decisions below. +_MESSAGES_ROUTABLE_SELECTORS = [ + selector for selector in anthropic_model_recommendation.MANTLE_SELECTORS + if selector not in {"background_inference", "projects"} +] + + +@pytest.mark.parametrize("selector", _MESSAGES_ROUTABLE_SELECTORS) def test_every_mantle_selector_selects_mantle_messages_and_is_echoed(selector): rec = _recommend(_workload(requirements={"mantle_selector": selector})) @@ -109,6 +116,75 @@ def test_every_mantle_selector_selects_mantle_messages_and_is_echoed(selector): assert any(f"mantle_selector {selector}" in line for line in rec["rationale"]) +# --- Every selector is validated against the final provider and API (PR 588 F4) --- +# Background inference is a Responses API feature on bedrock-mantle +# (inference-responses-api.html, read 2026-11-09); `projects` scopes the +# OpenAI-compatible APIs while Messages scopes by Workspaces (projects.html, +# workspaces.html, read 2026-11-09). + +def test_anthropic_messages_with_background_inference_is_a_decision_not_mantle_messages(): + workload = _workload(requirements={"mantle_selector": "background_inference"}) + result = model_recommendation.recommend(_input(workload)) + jsonschema.validate(result, _schema("model-recommendation.json")) + rec = result["workloads"]["support-agent"] + + assert rec["decision_status"] == "decision_required" + assert rec["api_path"] is None + assert rec["mantle_selector"] is None + assert "selector_api_incompatible" in _codes(rec["blocks"]) + assert "mantle_selector_vs_anthropic_api" in _decision_block(rec)["message"] + # Both options stay on bedrock-runtime; mantle_messages is never offered. + assert {option["api_path"] for option in rec["decision_options"]} == { + "runtime_messages", "runtime_converse", + } + block = next(b for b in rec["blocks"] if b["code"] == "selector_api_incompatible") + assert "inference-responses-api.html" in block["message"] + assert "Responses" in block["message"] + + +def test_projects_selector_on_an_anthropic_source_is_a_provider_mismatch_decision(): + workload = _workload(requirements={"mantle_selector": "projects"}) + result = model_recommendation.recommend(_input(workload)) + jsonschema.validate(result, _schema("model-recommendation.json")) + rec = result["workloads"]["support-agent"] + + assert rec["decision_status"] == "decision_required" + assert rec["api_path"] is None + assert "selector_provider_mismatch" in _codes(rec["blocks"]) + assert "mantle_selector_provider_mismatch" in _decision_block(rec)["message"] + assert {option["api_path"] for option in rec["decision_options"]} == { + "runtime_messages", "runtime_converse", + } + block = next(b for b in rec["blocks"] if b["code"] == "selector_provider_mismatch") + assert "anthropic-workspace-id" in block["message"] + assert "OpenAI-Project" in block["message"] + + +def test_projects_with_a_preferred_mantle_path_still_never_offers_mantle_messages(): + rec = _recommend( + _workload(requirements={"preferred_api_path": "mantle_messages", + "mantle_selector": "projects"}) + ) + + assert rec["decision_status"] == "decision_required" + assert "selector_provider_mismatch" in _codes(rec["blocks"]) + assert not any(o["api_path"].startswith("mantle_") for o in rec["decision_options"]) + + +def test_background_inference_with_native_payload_offers_runtime_messages_or_invoke(): + rec = _recommend( + _workload(requirements={"mantle_selector": "background_inference", + "requires_native_payload": True}) + ) + + assert rec["decision_status"] == "decision_required" + assert "selector_api_incompatible" in _codes(rec["blocks"]) + assert "mantle_selector_vs_runtime" not in _decision_block(rec)["message"] + assert {option["api_path"] for option in rec["decision_options"]} == { + "runtime_messages", "runtime_invoke", + } + + def test_mantle_selectors_match_the_input_schema_enum(): enum = _schema("model-recommendation-input.json")["properties"]["workloads"]["items"][ "properties"]["requirements"]["properties"]["mantle_selector"]["enum"] @@ -320,12 +396,12 @@ def test_preferred_mantle_path_without_selector_fails_closed(): def test_preferred_mantle_path_with_selector_is_honored(): rec = _recommend( _workload(requirements={"preferred_api_path": "mantle_messages", - "mantle_selector": "projects"}) + "mantle_selector": "workspaces"}) ) assert rec["decision_status"] == "recommended" assert rec["api_path"] == "mantle_messages" - assert rec["mantle_selector"] == "projects" + assert rec["mantle_selector"] == "workspaces" def test_preferred_runtime_messages_is_honored_and_does_not_conflict_with_required_messages(): diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py index 7722de8cb..00ba660e8 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py @@ -145,7 +145,8 @@ def test_continuity_plus_logging_governance_stays_on_runtime_openai_api(): assert rec["primary_model"] == "openai.gpt-5.4" -@pytest.mark.parametrize("selector", oai.MANTLE_SELECTORS) +# `workspaces` scopes the Anthropic Messages API and is a provider-mismatch decision below. +@pytest.mark.parametrize("selector", [s for s in oai.MANTLE_SELECTORS if s != "workspaces"]) def test_each_mantle_selector_selects_mantle_and_is_echoed(selector): rec = _recommend(_workload(requirements={"mantle_selector": selector})) assert rec["decision_status"] == "recommended" @@ -677,12 +678,87 @@ def test_preferred_mantle_chat_with_background_inference_is_a_decision(): assert _option(rec, "mantle_openai_responses")["mantle_selector"] == "background_inference" +# --- Web Search is a bedrock-mantle Responses API tool (web-search.html, read +# 2026-11-09); every selector is validated against the final provider and API ---- + +def test_gpt55_chat_source_with_web_search_is_a_decision_between_mantle_responses_and_runtime_chat(): + out = _full_recommend( + _workload(source={"model_ids": ["gpt-5.5"], "api_surface": "chat_completions"}, + requirements={"critical_features": ["web_search"]}), + region="us-east-1", + ) + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == "decision_required" + assert rec["api_path"] is None + assert "selector_api_incompatible" in _codes(rec["blocks"]) + assert _cites_aws_docs(rec["blocks"][0]["message"]) + assert "web-search.html" in rec["blocks"][0]["message"] + assert {o["api_path"] for o in rec["decision_options"]} == { + "mantle_openai_responses", "runtime_openai_chat", + } + assert _option(rec, "mantle_openai_responses")["mantle_selector"] == "web_search" + assert _option(rec, "runtime_openai_chat")["mantle_selector"] is None + assert not any(o["api_path"] == "mantle_openai_chat" for o in rec["decision_options"]) + + +def test_gpt55_chat_with_web_search_and_free_continuity_reshapes_to_mantle_responses(): + rec = _recommend( + _workload(source={"model_ids": ["gpt-5.5"], "api_surface": "chat_completions"}, + requirements={"critical_features": ["web_search"], + "api_continuity": "not_required"}), + region="us-east-1", + ) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "mantle_openai_responses" + assert rec["primary_model"] == "openai.gpt-5.5" + assert rec["mantle_selector"] == "web_search" + assert "chat_completions_to_responses_required" in _codes(rec["blocks"]) + assert "web_search" in rec["compatibility"]["portable"] + + +def test_chat_source_with_web_search_plus_logging_never_offers_a_mantle_chat_option(): + out = _full_recommend(_workload( + source={"model_ids": ["gpt-5.5"], "api_surface": "chat_completions"}, + requirements={"critical_features": ["web_search"], + "governance": ["invocation_logging"]}, + ), region="us-east-1") + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == "decision_required" + assert {o["api_path"] for o in rec["decision_options"]} == { + "mantle_openai_responses", "runtime_openai_chat", + } + assert _option(rec, "mantle_openai_responses")["mantle_selector"] == "web_search" + + +def test_workspaces_selector_on_an_openai_source_is_a_provider_mismatch_decision(): + out = _full_recommend(_workload(requirements={"mantle_selector": "workspaces"})) + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + assert rec["decision_status"] == "decision_required" + assert rec["api_path"] is None + assert rec["mantle_selector"] is None + assert "selector_provider_mismatch" in _codes(rec["blocks"]) + block = next(b for b in rec["blocks"] if b["code"] == "selector_provider_mismatch") + assert _cites_aws_docs(block["message"]) + assert "anthropic-workspace-id" in block["message"] + assert "`projects`" in block["message"] + # Both options stay on bedrock-runtime: the source API, or Converse governance. + assert {o["api_path"] for o in rec["decision_options"]} == { + "runtime_openai_responses", "runtime_converse", + } + assert not any(o["api_path"].startswith("mantle_") for o in rec["decision_options"]) + + # --- GPT-6.1 Sol lists server-side system tools as unsupported on bedrock-mantle # (model-card-openai-gpt-6-1-sol.html, read 2026-11-08) ----------------------- @pytest.mark.parametrize(("surface", "selector", "path"), [ ("responses", "server_side_tools", "mantle_openai_responses"), - ("chat_completions", "web_search", "mantle_openai_chat"), + ("chat_completions", "server_side_tools", "mantle_openai_chat"), + # Web Search is Responses-only: the Chat source reshapes once continuity is free. + ("chat_completions", "web_search", "mantle_openai_responses"), ]) def test_gpt61_sol_server_side_tool_selectors_fall_to_astra_and_keep_the_selector( surface, selector, path @@ -690,6 +766,8 @@ def test_gpt61_sol_server_side_tool_selectors_fall_to_astra_and_keep_the_selecto requirements = {"mantle_selector": selector} if surface == "responses": requirements["api_continuity"] = "required" + if selector == "web_search": + requirements["api_continuity"] = "not_required" out = _full_recommend( _workload(source={"model_ids": ["gpt-6.1-sol"], "api_surface": surface}, requirements=requirements), From 26970991a3e8f41892d91bad872bf3234a6e3abd Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 9 Nov 2026 11:04:30 -0800 Subject: [PATCH 39/41] fix(aws-startup-advisor): agent-advisor checks required prompt caching on the selected API for every candidate via the catalog prompt_caching_supported flag (PR 588 F14) Catalog (skills/shared/ai/models/openai-bedrock.json + agent-advisor vendored copy): every GPT path row gains a boolean `prompt_caching_supported` with the card read in its evidence (read 2026-11-09). GPT-5.5 / GPT-5.4: implicit caching on the bedrock-runtime Responses API only (Chat, Converse and both Mantle rows false). GPT-5.6 Sol/Terra/Luna: every runtime API true, Mantle Responses true, Mantle Chat false (the 2026-08-21 "Responses-only" Converse note is marked superseded). GPT-6 Sol / Luna / 6.1 Sol: all rows true. GPT-6 Astra mirrors its existing per-path capabilities, so no PR 1/2/3 fact changes. Engine (openai_model_recommendation.py): `_evidenced_capabilities` adds prompt_caching to a row's capabilities iff its flag is true; `_unsupported_required_capabilities` and `_compatibility` take `path_config` and check prompt_caching generically (the Astra display-name special case is gone). `_catalog_model_for_path` and the final record pass the selected row, so the candidate walk skips a candidate whose selected-API row lacks caching (GPT-5.5 Chat + required continuity + critical prompt_caching lands on the same-family GPT-5.6 Chat row with a rationale naming the caching scope instead of a false "unavailable"); when no candidate's row caches, the `unverified_capability` finding names the Chat-versus-Responses continuity trade-off (`_other_runtime_api_with_prompt_caching`). Never a clean runtime_openai_chat recommendation with empty buckets. Docs: model-recommend.md and ai-openai-to-bedrock.md (+2 vendored) carry the flag and the per-model scope. Tests: test_catalog_paths.py::test_every_gpt_path_row_carries_a_boolean_prompt_caching_flag; test_openai_model_recommendation.py::{test_gpt55_chat_with_required_continuity_and_critical_prompt_caching_never_recommends_a_cache_less_row, test_gpt55_responses_with_critical_prompt_caching_is_recommended_natively, test_prompt_caching_decision_names_the_responses_tradeoff_when_no_chat_candidate_has_it, test_gpt55_chat_without_a_caching_requirement_is_still_a_clean_recommendation}; test_astra_required_caching_is_checked_on_the_selected_api unchanged and green. Validation (this commit): skills/agent-advisor/scripts pytest 800 passed (789 at 91827f91); skills/llm-to-bedrock/scripts pytest 672 passed; plugin tests 1532 passed / 1 failed (known tests/test_model_id_lint.py::test_real_plugin_passes on the untracked skills/llm-to-bedrock/scripts/.venv anthropic/_constants.py); validate-artifacts.py --fixtures PASS (103); sync-vendored --check PASS (65); check-skill-refs --check PASS (1544); model-catalog-drift exit 0 (5 warnings, pre-existing); git diff --check clean; markdownlint 0 errors on touched docs; `mise run ci` fails only at lint:md with 75 errors all under the untracked .venv (0 outside). gitleaks (`mise run build`) not re-run: known pre-existing failure on upstream f8d09fd1. --- .../phases/model-recommend/model-recommend.md | 13 +- .../vendored/ai/models/openai-bedrock.json | 120 +++++++++++------- .../scripts/openai_model_recommendation.py | 110 ++++++++++++---- .../scripts/test_catalog_paths.py | 48 +++++++ .../test_openai_model_recommendation.py | 87 +++++++++++++ .../vendored/ai/ai-openai-to-bedrock.md | 9 +- .../vendored/ai/ai-openai-to-bedrock.md | 9 +- .../skills/shared/ai/ai-openai-to-bedrock.md | 9 +- .../shared/ai/models/openai-bedrock.json | 120 +++++++++++------- 9 files changed, 409 insertions(+), 116 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md index c9fffd327..a4949f9b9 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md @@ -122,7 +122,18 @@ GPT-6.1 Sol keeps `runtime_openai_chat`, because their cards list Guardrails on catalog row flag `guardrails_supported`; cards read 2026-11-08: `https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html`, `model-card-openai-gpt-6-luna.html`, `model-card-openai-gpt-6-1-sol.html`). A GPT-6.1 Sol -Responses source with Guardrails is a `decision_required` because its card excludes Responses. A +Responses source with Guardrails is a `decision_required` because its card excludes Responses. +The catalog row flag `prompt_caching_supported` carries the per-API prompt-caching scope the same way +(GPT-5.5 / GPT-5.4: implicit caching on the bedrock-runtime Responses API only, none on +`bedrock-mantle`; GPT-5.6 Sol / Terra / Luna: every runtime API and Mantle Responses only; GPT-6 Sol / +Luna and GPT-6.1 Sol: every API on both endpoints; cards read 2026-11-09: +`https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html`, +`model-card-openai-gpt-54.html`). A required `prompt_caching` is checked on the selected API row for +every candidate: a GPT-5.5 Chat Completions source with required continuity and critical +`prompt_caching` lands on the same-family candidate whose Chat row caches (GPT-5.6) with the reason +in `rationale`, or — when no candidate's row caches — is an `unverified_capability` decision that +names the Chat-versus-Responses continuity trade-off. It is never a clean `runtime_openai_chat` +recommendation with empty compatibility buckets. A shared multi-model Converse surface always selects `runtime_converse`. Required API continuity plus Converse-only governance is a `decision_required` between two bedrock-runtime options (`runtime_openai_` and `runtime_converse`); Mantle is never offered without a selector. diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json index bae98f09f..1ba299cdb 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json @@ -142,35 +142,40 @@ "model_id": "openai.gpt-5.6-sol", "requires_cris": false, "guardrails_supported": false, - "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard" + "prompt_caching_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard; prompt_caching_supported true: the card lists implicit prompt caching on the Responses API only on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html, read 2026-11-09)" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": false, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2" + "prompt_caching_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2; prompt_caching_supported false: the card lists implicit prompt caching on the Responses API only on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html, read 2026-11-09)" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": true, "guardrails_supported": true, - "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): us. from six US/Canada source regions, global. from every listed commercial region; no eu./in./us-gov. profile" + "prompt_caching_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): us. from six US/Canada source regions, global. from every listed commercial region; no eu./in./us-gov. profile; prompt_caching_supported true: the card now lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime, superseding the 2026-08-21 Responses-only note (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html, read 2026-11-09)" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": true, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported (base URL bedrock-runtime.{region}.amazonaws.com/openai/v1); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol; the card's runtime table is the source of this model's inference_profiles matrix (us. and global. only); Guardrails Converse-only" + "prompt_caching_supported": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported (base URL bedrock-runtime.{region}.amazonaws.com/openai/v1); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol; the card's runtime table is the source of this model's inference_profiles matrix (us. and global. only); Guardrails Converse-only; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html, read 2026-11-09)" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": true, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported (/openai/v1/chat/completions); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol; the card's runtime table is the source of this model's inference_profiles matrix (us. and global. only); Guardrails Converse-only" + "prompt_caching_supported": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported (/openai/v1/chat/completions); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol; the card's runtime table is the source of this model's inference_profiles matrix (us. and global. only); Guardrails Converse-only; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html, read 2026-11-09)" } } }, @@ -209,38 +214,43 @@ "model_id": "openai.gpt-5.5", "requires_cris": false, "guardrails_supported": false, - "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1 and us-east-2 only; Geo/Global not supported on mantle" + "prompt_caching_supported": false, + "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1 and us-east-2 only; Geo/Global not supported on mantle; prompt_caching_supported false: the card's bedrock-mantle feature list names no prompt caching (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, read 2026-11-09)" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": false, "guardrails_supported": false, - "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code" + "prompt_caching_supported": false, + "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code; prompt_caching_supported false: the card's bedrock-mantle feature list names no prompt caching (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, read 2026-11-09)" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": true, "guardrails_supported": true, + "prompt_caching_supported": false, "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision", "streaming"], - "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.5 (Geo) or global.openai.gpt-5.5 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): 'Response streaming' listed for runtime; 'Structured outputs (JSON Schema; Responses API)' is scoped to Responses, so structured_output_json is not carried on Converse; Guardrails Converse-only" + "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.5 (Geo) or global.openai.gpt-5.5 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): 'Response streaming' listed for runtime; 'Structured outputs (JSON Schema; Responses API)' is scoped to Responses, so structured_output_json is not carried on Converse; Guardrails Converse-only; prompt_caching_supported false: the card lists 'Implicit Prompt Caching (Responses API only; automatic, no request parameters)' on bedrock-runtime and Explicit Prompt Caching as not supported (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, read 2026-11-09)" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": true, "guardrails_supported": false, + "prompt_caching_supported": true, "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision", "structured_output_json", "streaming"], - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): bedrock-runtime lists Responses supported; 'Structured outputs (JSON Schema; Responses API)' and 'Response streaming' listed for runtime, implicit prompt caching Responses-only; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): bedrock-runtime lists Responses supported; 'Structured outputs (JSON Schema; Responses API)' and 'Response streaming' listed for runtime, implicit prompt caching Responses-only; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5; prompt_caching_supported true: the card lists 'Implicit Prompt Caching (Responses API only; automatic, no request parameters)' on bedrock-runtime and Explicit Prompt Caching as not supported (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, read 2026-11-09)" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": true, "guardrails_supported": false, + "prompt_caching_supported": false, "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision", "streaming"], - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; 'Response streaming' listed for runtime; the card scopes 'Structured outputs (JSON Schema; Responses API)' and implicit prompt caching to Responses, so structured_output_json is not carried on Chat Completions; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; 'Response streaming' listed for runtime; the card scopes 'Structured outputs (JSON Schema; Responses API)' and implicit prompt caching to Responses, so structured_output_json is not carried on Chat Completions; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5; prompt_caching_supported false: the card lists 'Implicit Prompt Caching (Responses API only; automatic, no request parameters)' on bedrock-runtime and Explicit Prompt Caching as not supported (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, read 2026-11-09)" } } }, @@ -281,35 +291,40 @@ "model_id": "openai.gpt-5.6-terra", "requires_cris": false, "guardrails_supported": false, - "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard" + "prompt_caching_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard; prompt_caching_supported true: the card lists implicit prompt caching on the Responses API only on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html, read 2026-11-09)" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": false, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2, us-west-2, us-gov-west-1, us-gov-east-1" + "prompt_caching_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2, us-west-2, us-gov-west-1, us-gov-east-1; prompt_caching_supported false: the card lists implicit prompt caching on the Responses API only on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html, read 2026-11-09)" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": true, "guardrails_supported": true, - "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): us. from six US/Canada source regions, in. from ap-south-1/ap-south-2, us-gov. from us-gov-east-1/us-gov-west-1 (Global not supported there), global. from every listed commercial region" + "prompt_caching_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): us. from six US/Canada source regions, in. from ap-south-1/ap-south-2, us-gov. from us-gov-east-1/us-gov-west-1 (Global not supported there), global. from every listed commercial region; prompt_caching_supported true: the card now lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime, superseding the 2026-08-21 Responses-only note (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html, read 2026-11-09)" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": true, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" + "prompt_caching_supported": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html, read 2026-11-09)" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": true, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" + "prompt_caching_supported": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html, read 2026-11-09)" } } }, @@ -350,35 +365,40 @@ "model_id": "openai.gpt-5.6-luna", "requires_cris": false, "guardrails_supported": false, - "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard" + "prompt_caching_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard; prompt_caching_supported true: the card lists implicit prompt caching on the Responses API only on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html, read 2026-11-09)" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": false, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2, us-west-2, us-gov-west-1, us-gov-east-1" + "prompt_caching_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2, us-west-2, us-gov-west-1, us-gov-east-1; prompt_caching_supported false: the card lists implicit prompt caching on the Responses API only on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html, read 2026-11-09)" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": true, "guardrails_supported": true, - "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): us. from six US/Canada source regions, in. from ap-south-1/ap-south-2, us-gov. from us-gov-east-1/us-gov-west-1 (Global not supported there), global. from every listed commercial region" + "prompt_caching_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): us. from six US/Canada source regions, in. from ap-south-1/ap-south-2, us-gov. from us-gov-east-1/us-gov-west-1 (Global not supported there), global. from every listed commercial region; prompt_caching_supported true: the card now lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime, superseding the 2026-08-21 Responses-only note (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html, read 2026-11-09)" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": true, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" + "prompt_caching_supported": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html, read 2026-11-09)" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": true, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" + "prompt_caching_supported": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html, read 2026-11-09)" } } }, @@ -420,39 +440,44 @@ "model_id": "openai.gpt-5.4", "requires_cris": false, "guardrails_supported": false, - "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1, us-east-2, us-west-2 and us-gov-west-1; Geo/Global not supported on mantle" + "prompt_caching_supported": false, + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1, us-east-2, us-west-2 and us-gov-west-1; Geo/Global not supported on mantle; prompt_caching_supported false: the card's bedrock-mantle feature list names no prompt caching (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, read 2026-11-09)" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": false, "guardrails_supported": false, + "prompt_caching_supported": false, "capabilities": ["tool_or_function_calling", "reasoning", "streaming", "image_input_vision"], - "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): structured outputs are evidenced on the Responses API only, so structured_output_json is not carried on Mantle Chat Completions" + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): structured outputs are evidenced on the Responses API only, so structured_output_json is not carried on Mantle Chat Completions; prompt_caching_supported false: the card's bedrock-mantle feature list names no prompt caching (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, read 2026-11-09)" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": true, "guardrails_supported": true, + "prompt_caching_supported": false, "capabilities": ["tool_or_function_calling", "reasoning", "streaming", "image_input_vision"], - "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.4 (Geo) or global.openai.gpt-5.4 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): 'Response streaming' listed for runtime; 'Structured outputs (JSON Schema; Responses API)' is scoped to Responses, so structured_output_json is not carried on Converse; Guardrails Converse-only" + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.4 (Geo) or global.openai.gpt-5.4 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): 'Response streaming' listed for runtime; 'Structured outputs (JSON Schema; Responses API)' is scoped to Responses, so structured_output_json is not carried on Converse; Guardrails Converse-only; prompt_caching_supported false: the card lists 'Implicit Prompt Caching (Responses API only; automatic, no request parameters)' on bedrock-runtime and Explicit Prompt Caching as not supported (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, read 2026-11-09)" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": true, "guardrails_supported": false, + "prompt_caching_supported": true, "capabilities": ["tool_or_function_calling", "structured_output_json", "reasoning", "streaming", "image_input_vision"], - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): bedrock-runtime lists Responses supported; 'Structured outputs (JSON Schema; Responses API)' and 'Response streaming' listed for runtime, implicit prompt caching Responses-only; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): bedrock-runtime lists Responses supported; 'Structured outputs (JSON Schema; Responses API)' and 'Response streaming' listed for runtime, implicit prompt caching Responses-only; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4; prompt_caching_supported true: the card lists 'Implicit Prompt Caching (Responses API only; automatic, no request parameters)' on bedrock-runtime and Explicit Prompt Caching as not supported (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, read 2026-11-09)" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": true, "guardrails_supported": false, + "prompt_caching_supported": false, "capabilities": ["tool_or_function_calling", "reasoning", "streaming", "image_input_vision"], - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; 'Response streaming' listed for runtime; the card scopes 'Structured outputs (JSON Schema; Responses API)' and implicit prompt caching to Responses, so structured_output_json is not carried on Chat Completions; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; 'Response streaming' listed for runtime; the card scopes 'Structured outputs (JSON Schema; Responses API)' and implicit prompt caching to Responses, so structured_output_json is not carried on Chat Completions; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4; prompt_caching_supported false: the card lists 'Implicit Prompt Caching (Responses API only; automatic, no request parameters)' on bedrock-runtime and Explicit Prompt Caching as not supported (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, read 2026-11-09)" } } }, @@ -511,6 +536,7 @@ "model_id": "openai.gpt-6-astra", "requires_cris": false, "guardrails_supported": false, + "prompt_caching_supported": true, "supported_regions": [ "us-east-1", "us-west-2" @@ -521,13 +547,14 @@ "tool_or_function_calling", "prompt_caching" ], - "evidence": "AWS model card 2026-09-30: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html. Responses supported at /openai/v1/responses; Standard mantle is available in us-east-1 and us-west-2. Server-side tool calling and implicit/explicit Responses caching supported; application inference profiles unsupported." + "evidence": "AWS model card 2026-09-30: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html. Responses supported at /openai/v1/responses; Standard mantle is available in us-east-1 and us-west-2. Server-side tool calling and implicit/explicit Responses caching supported; application inference profiles unsupported.; prompt_caching_supported mirrors this row's cataloged capabilities (prompt_caching present or absent; https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html, read 2026-11-09)" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": false, "guardrails_supported": false, + "prompt_caching_supported": false, "supported_regions": [ "us-east-1", "us-west-2" @@ -536,13 +563,14 @@ "reasoning", "image_input_vision" ], - "evidence": "AWS model card 2026-09-30: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html. Chat Completions supported at /openai/v1/chat/completions in us-east-1 and us-west-2; prompt caching is Responses-only. Tool behavior on Chat Completions remains unprobed." + "evidence": "AWS model card 2026-09-30: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html. Chat Completions supported at /openai/v1/chat/completions in us-east-1 and us-west-2; prompt caching is Responses-only. Tool behavior on Chat Completions remains unprobed.; prompt_caching_supported mirrors this row's cataloged capabilities (prompt_caching present or absent; https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html, read 2026-11-09)" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": true, "guardrails_supported": true, + "prompt_caching_supported": false, "supported_regions": [ "us-east-1", "us-east-2", @@ -596,13 +624,14 @@ "streaming", "structured_output_json" ], - "evidence": "AWS model card, runtime JSON Schema reverified 2026-10-03: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html. Runtime APIs: Responses, Chat Completions, Converse; Invoke and Messages unsupported. US Geo/Global CRIS only, no in-region or EU/India Geo form. Guardrails and application inference profiles are Converse-only. Non-streaming JSON Schema outputs supported: Converse requires outputConfig.textFormat.structure.jsonSchema and additionalModelRequestFields.text.format.strict=true. Streaming structured output and server-side tool use are not evidenced. Runtime TPM accounting uses 10x output-token burndown." + "evidence": "AWS model card, runtime JSON Schema reverified 2026-10-03: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html. Runtime APIs: Responses, Chat Completions, Converse; Invoke and Messages unsupported. US Geo/Global CRIS only, no in-region or EU/India Geo form. Guardrails and application inference profiles are Converse-only. Non-streaming JSON Schema outputs supported: Converse requires outputConfig.textFormat.structure.jsonSchema and additionalModelRequestFields.text.format.strict=true. Streaming structured output and server-side tool use are not evidenced. Runtime TPM accounting uses 10x output-token burndown.; prompt_caching_supported mirrors this row's cataloged capabilities (prompt_caching present or absent; https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html, read 2026-11-09)" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": true, "guardrails_supported": false, + "prompt_caching_supported": true, "supported_regions": [ "us-east-1", "us-east-2", @@ -657,13 +686,14 @@ "structured_output_json", "prompt_caching" ], - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html (read 2026-11-07): bedrock-runtime lists Responses supported; runtime features: response streaming, structured outputs (JSON Schema), implicit and explicit prompt caching on Responses, reasoning summaries; application inference profiles are InvokeModel/Converse-only and server-side tool use is unsupported; no in-region form; Geo id us.openai.gpt-6-astra, Global id global.openai.gpt-6-astra" + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html (read 2026-11-07): bedrock-runtime lists Responses supported; runtime features: response streaming, structured outputs (JSON Schema), implicit and explicit prompt caching on Responses, reasoning summaries; application inference profiles are InvokeModel/Converse-only and server-side tool use is unsupported; no in-region form; Geo id us.openai.gpt-6-astra, Global id global.openai.gpt-6-astra; prompt_caching_supported mirrors this row's cataloged capabilities (prompt_caching present or absent; https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html, read 2026-11-09)" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": true, "guardrails_supported": false, + "prompt_caching_supported": true, "supported_regions": [ "us-east-1", "us-east-2", @@ -718,7 +748,7 @@ "structured_output_json", "prompt_caching" ], - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; runtime features: response streaming, structured outputs (JSON Schema), implicit and explicit prompt caching on Chat Completions; application inference profiles are InvokeModel/Converse-only and server-side tool use is unsupported; no in-region form; Geo id us.openai.gpt-6-astra, Global id global.openai.gpt-6-astra" + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; runtime features: response streaming, structured outputs (JSON Schema), implicit and explicit prompt caching on Chat Completions; application inference profiles are InvokeModel/Converse-only and server-side tool use is unsupported; no in-region form; Geo id us.openai.gpt-6-astra, Global id global.openai.gpt-6-astra; prompt_caching_supported mirrors this row's cataloged capabilities (prompt_caching present or absent; https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html, read 2026-11-09)" } } }, @@ -742,11 +772,11 @@ "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision"], "capability_source": "AWS GPT-6 Sol model card, retrieved 2026-10-10: text and image input, tool calling, 1.05M context and 128K output.", "paths": { - "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, - "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported. (Guardrails: Responses/Chat Completions/InvokeModel/Converse per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html; card read 2026-11-08)"}, - "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"}, - "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"} + "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "guardrails_supported": false, "prompt_caching_supported": true, "evidence": "AWS model card: Mantle Responses in us-east-1.; prompt_caching_supported true: the card lists implicit prompt caching on Responses and Chat Completions on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html, read 2026-11-09)"}, + "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "guardrails_supported": false, "prompt_caching_supported": true, "evidence": "AWS model card: Mantle Chat Completions supported.; prompt_caching_supported true: the card lists implicit prompt caching on Responses and Chat Completions on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html, read 2026-11-09)"}, + "runtime_converse": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported. (Guardrails: Responses/Chat Completions/InvokeModel/Converse per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html; card read 2026-11-08); prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html, read 2026-11-09)"}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html, read 2026-11-09)"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html, read 2026-11-09)"} } }, "openai_gpt_6_luna": { @@ -769,11 +799,11 @@ "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision"], "capability_source": "AWS GPT-6 Luna model card, retrieved 2026-10-10: text and image input, tool calling, 1.05M context and 128K output.", "paths": { - "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, - "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke. (Guardrails: Responses/Chat Completions/InvokeModel/Converse per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html; card read 2026-11-08)"}, - "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"}, - "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"} + "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "guardrails_supported": false, "prompt_caching_supported": true, "evidence": "AWS model card: Mantle Responses in us-east-1.; prompt_caching_supported true: the card lists implicit prompt caching on Responses and Chat Completions on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html, read 2026-11-09)"}, + "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "guardrails_supported": false, "prompt_caching_supported": true, "evidence": "AWS model card: Mantle Chat Completions supported.; prompt_caching_supported true: the card lists implicit prompt caching on Responses and Chat Completions on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html, read 2026-11-09)"}, + "runtime_converse": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke. (Guardrails: Responses/Chat Completions/InvokeModel/Converse per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html; card read 2026-11-08); prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html, read 2026-11-09)"}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html, read 2026-11-09)"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html, read 2026-11-09)"} } }, "openai_gpt_6_1_sol": { @@ -796,11 +826,11 @@ "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision"], "capability_source": "AWS GPT-6.1 Sol model card, retrieved 2026-10-10: text and image input, tool calling, 1.05M context and 131072 output.", "paths": { - "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "guardrails_supported": false, "server_side_tools_supported": false, "evidence": "AWS model card: Mantle Responses in us-east-1. https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): Server-side system tools listed as Not Supported on bedrock-mantle"}, - "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "guardrails_supported": false, "server_side_tools_supported": false, "evidence": "AWS model card: Mantle Chat Completions supported. https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): Server-side system tools listed as Not Supported on bedrock-mantle"}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported. (Guardrails: Chat Completions/InvokeModel/Converse, not Responses, per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html; card read 2026-11-08)"}, - "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": false, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails are listed for Chat Completions/InvokeModel/Converse but not Responses, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"}, - "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails listed on Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"} + "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "guardrails_supported": false, "prompt_caching_supported": true, "server_side_tools_supported": false, "evidence": "AWS model card: Mantle Responses in us-east-1. https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): Server-side system tools listed as Not Supported on bedrock-mantle; prompt_caching_supported true: the card lists implicit prompt caching on Responses and Chat Completions on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html, read 2026-11-09)"}, + "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "guardrails_supported": false, "prompt_caching_supported": true, "server_side_tools_supported": false, "evidence": "AWS model card: Mantle Chat Completions supported. https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): Server-side system tools listed as Not Supported on bedrock-mantle; prompt_caching_supported true: the card lists implicit prompt caching on Responses and Chat Completions on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html, read 2026-11-09)"}, + "runtime_converse": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported. (Guardrails: Chat Completions/InvokeModel/Converse, not Responses, per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html; card read 2026-11-08); prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html, read 2026-11-09)"}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": false, "prompt_caching_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails are listed for Chat Completions/InvokeModel/Converse but not Responses, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html, read 2026-11-09)"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails listed on Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html, read 2026-11-09)"} } }, "anthropic_claude_sonnet_5": { diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py index bb945ef12..371f84b2e 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py @@ -237,6 +237,29 @@ def _guardrails_on_openai_api(catalog, source, openai_path): return row.get("guardrails_supported") is True +def _other_runtime_api_with_prompt_caching(catalog, source, path): + """The recognized source model's OTHER bedrock-runtime OpenAI API when its + catalog row carries `prompt_caching_supported: true` and the selected `path` + row does not (GPT-5.5 / GPT-5.4 Chat sources: Responses caches, Chat does not). + None when `path` is not a runtime OpenAI API or no such row exists.""" + if path not in {"runtime_openai_responses", "runtime_openai_chat"}: + return None + same = _same_model_catalog_key(source) + if not same or catalog is None: + return None + paths = catalog["models"].get(same, {}).get("paths", {}) + other = ( + "runtime_openai_chat" if path == "runtime_openai_responses" else "runtime_openai_responses" + ) + if ( + paths.get(path, {}).get("prompt_caching_supported") is False + and paths.get(other, {}).get("available") is True + and paths.get(other, {}).get("prompt_caching_supported") is True + ): + return other + return None + + def _converse_required(requirements, catalog=None, source=None, openai_path=None): """Converse is required by multi-model Converse, by an application inference profile, and by Guardrails when the source model's card scopes them to Converse @@ -405,7 +428,7 @@ def _catalog_model_for_path(catalog, path, detected_features=None, requirements= ): continue unmet = _unsupported_required_capabilities( - model, detected_features, requirements or {} + model, detected_features, requirements or {}, path_config ) if first is None: first = (model_key, model, path_config) @@ -952,12 +975,26 @@ def _unknown_required_features(feature_assessment, requirements): return sorted(f for f in required if feature_assessment.get(f) == "unknown") -def _unsupported_required_capabilities(target_model, detected_features, requirements): +def _evidenced_capabilities(target_model, path_config): + """The target's catalog capabilities for the selected row, plus `prompt_caching` + when that row's `prompt_caching_supported` flag is true. Prompt caching is scoped + per API by every GPT card (GPT-5.5 / GPT-5.4: implicit caching on the bedrock-runtime + Responses API only; cards read 2026-11-09), so it is evidenced on the selected row + only, never inherited from the model.""" + caps = set((target_model or {}).get("capabilities") or []) + if (path_config or {}).get("prompt_caching_supported") is True: + caps.add("prompt_caching") + return caps + + +def _unsupported_required_capabilities(target_model, detected_features, requirements, + path_config=None): """Required/detected text-model features NOT evidenced by the target's catalog - capabilities. Hosted tools/modalities are handled as architecture impacts, not here.""" + capabilities on the selected path row. Hosted tools/modalities are handled as + architecture impacts, not here.""" if target_model is None: return [] - catalog_caps = set(target_model.get("capabilities") or []) + catalog_caps = _evidenced_capabilities(target_model, path_config) needed = set(requirements.get("critical_features") or []) needed.update(detected_features or []) checkable = needed.intersection( @@ -967,24 +1004,25 @@ def _unsupported_required_capabilities(target_model, detected_features, requirem "streaming", "image_input_vision", "reasoning", + "prompt_caching", } ) - if target_model.get("display_name") == "OpenAI GPT-6 Astra (on Bedrock)" and "prompt_caching" in needed: - checkable.add("prompt_caching") return sorted(checkable - catalog_caps) -def _compatibility(detected_features, requirements, path, target_model=None): - """native = only features the SELECTED target's catalog evidences; features - needed but not evidenced go to unsupported (never derive native from the name).""" +def _compatibility(detected_features, requirements, path, target_model=None, path_config=None): + """native = only features the SELECTED target's catalog evidences on the selected + path row; features needed but not evidenced go to unsupported (never derive native + from the name).""" detected = set(detected_features or []) detected.update(requirements.get("critical_features") or []) text_features = detected.intersection( - {"tool_or_function_calling", "structured_output_json", "streaming", "image_input_vision", "reasoning"} + { + "tool_or_function_calling", "structured_output_json", "streaming", + "image_input_vision", "reasoning", "prompt_caching", + } ) - if (target_model or {}).get("display_name") == "OpenAI GPT-6 Astra (on Bedrock)" and "prompt_caching" in detected: - text_features.add("prompt_caching") - catalog_caps = set((target_model or {}).get("capabilities") or []) + catalog_caps = _evidenced_capabilities(target_model, path_config) native = sorted(text_features.intersection(catalog_caps)) if target_model else [] unsupported = sorted(text_features - catalog_caps) if target_model else [] hosted = detected.intersection(_HOSTED_TOOL_IMPACTS) @@ -1726,6 +1764,18 @@ def _preferred_path_conflict(mantle_selector, message): else " (bedrock-endpoint-policy.md default)." ) ) + elif ( + "prompt_caching" in (set(detected) | set(requirements.get("critical_features") or [])) + and _other_runtime_api_with_prompt_caching(catalog, source, path) + ): + same_name = catalog["models"][_same_model_catalog_key(source)]["display_name"] + rationale_head = ( + f"{same_name} lists implicit prompt caching on the Responses API only on " + "bedrock-runtime (model card, read 2026-11-09), so its Chat Completions row " + f"cannot evidence the required prompt_caching; {model['display_name']} is the " + f"same-family {endpoint} Chat Completions candidate whose card lists caching on " + "this API." + ) else: rationale_head = ( f"The exact source model is unavailable in {region}; " @@ -1750,20 +1800,36 @@ def _preferred_path_conflict(mantle_selector, message): f"that serves mantle_selector {selector}." ) if unsupported: + message = ( + f"No {path} candidate is evidenced to support required features: " + + ", ".join(unsupported) + ) + remediation = ( + "Add dated capability evidence for a candidate, choose a different path/model, " + "or drop the requirement, then rerun Model Recommend." + ) + caching_api = _other_runtime_api_with_prompt_caching(catalog, source, path) + if unsupported == ["prompt_caching"] and caching_api: + # The source model does cache on its other bedrock-runtime OpenAI API: the + # open decision is the API-continuity trade-off, not a missing model. + same_name = catalog["models"][_same_model_catalog_key(source)]["display_name"] + message += ( + f". {same_name} lists implicit prompt caching on the {_api_label(caching_api)} " + f"API only on bedrock-runtime (model card, read 2026-11-09); keep " + f"{_api_label(path)} without caching or reshape to {_api_label(caching_api)}." + ) + remediation = ( + f"Keep {_api_label(path)} on bedrock-runtime and drop prompt_caching, or set " + f"api_continuity to not_required (or preferred_api_path {caching_api}) to " + f"reshape to {_api_label(caching_api)}, then rerun Model Recommend." + ) return _unresolved( rationale_head, [ f"No {path} candidate has catalog capability evidence for: " + ", ".join(unsupported) ], - _finding( - "unverified_capability", - "[BLOCKS]", - f"No {path} candidate is evidenced to support required features: " - + ", ".join(unsupported), - "Add dated capability evidence for a candidate, choose a different path/model, " - "or drop the requirement, then rerun Model Recommend.", - ), + _finding("unverified_capability", "[BLOCKS]", message, remediation), path, ) @@ -1972,7 +2038,7 @@ def _preferred_path_conflict(mantle_selector, message): "rationale": rationale, "blocks": blocks, "tuning": tuning, - "compatibility": _compatibility(detected, requirements, path, model), + "compatibility": _compatibility(detected, requirements, path, model, path_config), "architecture_impacts": impacts, "additional_targets": _additional_targets(detected, requirements), "migration_deltas": deltas, diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_catalog_paths.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_catalog_paths.py index 90869d231..d2a9fc4e6 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_catalog_paths.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_catalog_paths.py @@ -190,6 +190,54 @@ def test_every_gpt_path_row_carries_a_boolean_guardrails_flag(catalogs): assert chat_true == {"openai_gpt_6_sol", "openai_gpt_6_luna", "openai_gpt_6_1_sol"} +def test_every_gpt_path_row_carries_a_boolean_prompt_caching_flag(catalogs): + # Per-API prompt-caching scope from the nine GPT cards (read 2026-11-09): GPT-5.5 / + # GPT-5.4 list implicit caching on the bedrock-runtime Responses API only and no + # caching on bedrock-mantle; GPT-5.6 Sol/Terra/Luna cache on every runtime API and + # on Mantle Responses only; GPT-6 Sol/Luna and GPT-6.1 Sol cache on every runtime + # API and on both Mantle APIs; GPT-6 Astra mirrors its per-path capabilities. + models = catalogs["openai"]["models"] + runtime_responses_true = set() + runtime_chat_true = set() + runtime_converse_true = set() + mantle_responses_true = set() + mantle_chat_true = set() + gpt_keys = set() + for model_key, model in models.items(): + if model.get("generation") == "bedrock_native": + continue + gpt_keys.add(model_key) + for path, row in model["paths"].items(): + assert isinstance(row.get("prompt_caching_supported"), bool), f"{model_key}.{path}" + assert "prompt_caching_supported" in row["evidence"], f"{model_key}.{path}" + assert "read 2026-11-09" in row["evidence"], f"{model_key}.{path}" + assert _cites_aws_docs(row["evidence"]), f"{model_key}.{path}" + if row["prompt_caching_supported"]: + { + "runtime_openai_responses": runtime_responses_true, + "runtime_openai_chat": runtime_chat_true, + "runtime_converse": runtime_converse_true, + "mantle_openai_responses": mantle_responses_true, + "mantle_openai_chat": mantle_chat_true, + }[path].add(model_key) + if "prompt_caching" in (row.get("capabilities") or []): + # A row whose capability list names prompt_caching (Astra) carries the flag. + assert row["prompt_caching_supported"] is True, f"{model_key}.{path}" + assert runtime_responses_true == gpt_keys + assert runtime_chat_true == { + "openai_gpt_5_6_sol", "openai_gpt_5_6_terra", "openai_gpt_5_6_luna", + "openai_gpt_6_sol", "openai_gpt_6_luna", "openai_gpt_6_1_sol", "openai_gpt_6_astra", + } + assert runtime_converse_true == { + "openai_gpt_5_6_sol", "openai_gpt_5_6_terra", "openai_gpt_5_6_luna", + "openai_gpt_6_sol", "openai_gpt_6_luna", "openai_gpt_6_1_sol", + } + assert mantle_responses_true == gpt_keys - {"openai_gpt_5_5", "openai_gpt_5_4"} + assert mantle_chat_true == {"openai_gpt_6_sol", "openai_gpt_6_luna", "openai_gpt_6_1_sol"} + for model_key in ("openai_gpt_5_5", "openai_gpt_5_4"): + assert "Responses API only" in models[model_key]["paths"]["runtime_openai_chat"]["evidence"] + + def test_only_gpt61_sol_mantle_rows_deny_server_side_tools(catalogs): models = catalogs["openai"]["models"] denied = { diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py index 00ba660e8..9677ea85e 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py @@ -7,6 +7,7 @@ and multi-model Converse select runtime Converse. """ +import copy import json import pathlib import re @@ -2060,6 +2061,92 @@ def test_astra_required_caching_is_checked_on_the_selected_api(region, surface, assert "prompt_caching" in rec["compatibility"]["native"] +# --- Required prompt caching is checked on the selected API for EVERY candidate via +# the catalog `prompt_caching_supported` row flag (GPT-5.5 / GPT-5.4 cards: implicit +# caching on the bedrock-runtime Responses API only; read 2026-11-09) ------------- + +def test_gpt55_chat_with_required_continuity_and_critical_prompt_caching_never_recommends_a_cache_less_row(): + out = _full_recommend( + _workload(source={"model_ids": ["gpt-5.5"], "api_surface": "chat_completions"}, + requirements={"api_continuity": "required", + "data_residency": "global_allowed", + "critical_features": ["prompt_caching"]}), + region="us-east-1", + ) + jsonschema.validate(out, _OUT_SCHEMA) + rec = out["workloads"]["chat-svc"] + # Never a clean GPT-5.5 Chat row with empty buckets. + assert not ( + rec["decision_status"] == "recommended" + and rec["primary_model"] == "openai.gpt-5.5" + ) + if rec["decision_status"] == "recommended": + assert rec["api_path"] == "runtime_openai_chat" + assert "prompt_caching" in rec["compatibility"]["native"] + assert rec["compatibility"]["unsupported"] == [] + row = OPENAI_CATALOG["models"][rec["model_identity"]["model_key"]]["paths"]["runtime_openai_chat"] + assert row["prompt_caching_supported"] is True + assert rec["model_identity"]["family"] == "openai_gpt_5" + # The fallback is explained by the caching scope, not a false "unavailable". + assert "prompt caching on the Responses API only" in rec["rationale"][0] + assert "unavailable" not in rec["rationale"][0] + else: + assert rec["decision_status"] == "decision_required" + assert "unverified_capability" in _codes(rec["blocks"]) + assert "prompt_caching" in rec["blocks"][0]["message"] + + +def test_gpt55_responses_with_critical_prompt_caching_is_recommended_natively(): + rec = _recommend( + _workload(source={"model_ids": ["gpt-5.5"], "api_surface": "responses"}, + requirements={"critical_features": ["prompt_caching"]}), + region="us-east-1", + ) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "runtime_openai_responses" + assert rec["primary_model"] == "openai.gpt-5.5" + assert "prompt_caching" in rec["compatibility"]["native"] + assert rec["compatibility"]["unsupported"] == [] + + +def test_prompt_caching_decision_names_the_responses_tradeoff_when_no_chat_candidate_has_it(): + catalog = copy.deepcopy(OPENAI_CATALOG) + catalog["models"] = { + key: model for key, model in catalog["models"].items() + if key == "openai_gpt_5_5" or model.get("generation") == "bedrock_native" + } + rec = oai.recommend_openai_workload( + _workload(source={"model_ids": ["gpt-5.5"], "api_surface": "chat_completions"}, + requirements={"api_continuity": "required", + "critical_features": ["prompt_caching"]}), + "us-east-1", catalog, + ) + assert rec["decision_status"] == "decision_required" + assert rec["api_path"] is None + assert "unverified_capability" in _codes(rec["blocks"]) + block = rec["blocks"][0] + assert "prompt_caching" in block["message"] + assert "Responses" in block["message"] + assert "GPT-5.5" in block["message"] + assert "read 2026-11-09" in block["message"] + assert "reshape to Responses" in block["message"] + assert "drop prompt_caching" in block["remediation"] + assert rec["compatibility"]["native"] == [] + + +def test_gpt55_chat_without_a_caching_requirement_is_still_a_clean_recommendation(): + rec = _recommend( + _workload(source={"model_ids": ["gpt-5.5"], "api_surface": "chat_completions"}, + requirements={"api_continuity": "required"}), + region="us-east-1", + ) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "runtime_openai_chat" + assert rec["primary_model"] == "openai.gpt-5.5" + assert "prompt_caching" not in rec["compatibility"]["native"] + assert rec["compatibility"]["unsupported"] == [] + + def test_astra_decision_options_ignore_an_explicitly_absent_cache_signal(): workload = _workload( source={"model_ids": ["gpt-6-astra"], "api_surface": "chat_completions"}, diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index 5dc0c6375..dce73fd38 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -411,7 +411,14 @@ is the default; the Bedrock-native path is the most feature-complete; the mantle recorded `mantle_selector`. Guardrails API scope is per model card (read 2026-11-08): GPT-6 Sol and GPT-6 Luna list Guardrails on Responses, Chat Completions, InvokeModel and Converse; GPT-6.1 Sol on Chat Completions, InvokeModel and Converse (not Responses); GPT-5.6 Sol/Terra/Luna, GPT-5.5, GPT-5.4 -and GPT-6 Astra on Converse only — the catalog row flag `guardrails_supported` carries this. +and GPT-6 Astra on Converse only — the catalog row flag `guardrails_supported` carries this. The row flag +`prompt_caching_supported` carries the per-API prompt-caching scope the same way (cards read 2026-11-09): +GPT-5.5 and GPT-5.4 list implicit caching on the `bedrock-runtime` Responses API only and none on +`bedrock-mantle`; GPT-5.6 Sol/Terra/Luna on every `bedrock-runtime` API and on `bedrock-mantle` Responses +only; GPT-6 Sol, GPT-6 Luna and GPT-6.1 Sol on every API of both endpoints. A required `prompt_caching` is +checked on the selected API row for every candidate, so a GPT-5.5 Chat Completions workload that needs +caching is never a clean same-API recommendation — it is a same-family candidate whose Chat row caches, or +an explicit Chat-versus-Responses continuity decision. ### High-spend tiering diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index 5dc0c6375..dce73fd38 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -411,7 +411,14 @@ is the default; the Bedrock-native path is the most feature-complete; the mantle recorded `mantle_selector`. Guardrails API scope is per model card (read 2026-11-08): GPT-6 Sol and GPT-6 Luna list Guardrails on Responses, Chat Completions, InvokeModel and Converse; GPT-6.1 Sol on Chat Completions, InvokeModel and Converse (not Responses); GPT-5.6 Sol/Terra/Luna, GPT-5.5, GPT-5.4 -and GPT-6 Astra on Converse only — the catalog row flag `guardrails_supported` carries this. +and GPT-6 Astra on Converse only — the catalog row flag `guardrails_supported` carries this. The row flag +`prompt_caching_supported` carries the per-API prompt-caching scope the same way (cards read 2026-11-09): +GPT-5.5 and GPT-5.4 list implicit caching on the `bedrock-runtime` Responses API only and none on +`bedrock-mantle`; GPT-5.6 Sol/Terra/Luna on every `bedrock-runtime` API and on `bedrock-mantle` Responses +only; GPT-6 Sol, GPT-6 Luna and GPT-6.1 Sol on every API of both endpoints. A required `prompt_caching` is +checked on the selected API row for every candidate, so a GPT-5.5 Chat Completions workload that needs +caching is never a clean same-API recommendation — it is a same-family candidate whose Chat row caches, or +an explicit Chat-versus-Responses continuity decision. ### High-spend tiering diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md index 5dc0c6375..dce73fd38 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md @@ -411,7 +411,14 @@ is the default; the Bedrock-native path is the most feature-complete; the mantle recorded `mantle_selector`. Guardrails API scope is per model card (read 2026-11-08): GPT-6 Sol and GPT-6 Luna list Guardrails on Responses, Chat Completions, InvokeModel and Converse; GPT-6.1 Sol on Chat Completions, InvokeModel and Converse (not Responses); GPT-5.6 Sol/Terra/Luna, GPT-5.5, GPT-5.4 -and GPT-6 Astra on Converse only — the catalog row flag `guardrails_supported` carries this. +and GPT-6 Astra on Converse only — the catalog row flag `guardrails_supported` carries this. The row flag +`prompt_caching_supported` carries the per-API prompt-caching scope the same way (cards read 2026-11-09): +GPT-5.5 and GPT-5.4 list implicit caching on the `bedrock-runtime` Responses API only and none on +`bedrock-mantle`; GPT-5.6 Sol/Terra/Luna on every `bedrock-runtime` API and on `bedrock-mantle` Responses +only; GPT-6 Sol, GPT-6 Luna and GPT-6.1 Sol on every API of both endpoints. A required `prompt_caching` is +checked on the selected API row for every candidate, so a GPT-5.5 Chat Completions workload that needs +caching is never a clean same-API recommendation — it is a same-family candidate whose Chat row caches, or +an explicit Chat-versus-Responses continuity decision. ### High-spend tiering diff --git a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json index bae98f09f..1ba299cdb 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json @@ -142,35 +142,40 @@ "model_id": "openai.gpt-5.6-sol", "requires_cris": false, "guardrails_supported": false, - "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard" + "prompt_caching_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard; prompt_caching_supported true: the card lists implicit prompt caching on the Responses API only on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html, read 2026-11-09)" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": false, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2" + "prompt_caching_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2; prompt_caching_supported false: the card lists implicit prompt caching on the Responses API only on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html, read 2026-11-09)" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": true, "guardrails_supported": true, - "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): us. from six US/Canada source regions, global. from every listed commercial region; no eu./in./us-gov. profile" + "prompt_caching_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): us. from six US/Canada source regions, global. from every listed commercial region; no eu./in./us-gov. profile; prompt_caching_supported true: the card now lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime, superseding the 2026-08-21 Responses-only note (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html, read 2026-11-09)" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": true, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported (base URL bedrock-runtime.{region}.amazonaws.com/openai/v1); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol; the card's runtime table is the source of this model's inference_profiles matrix (us. and global. only); Guardrails Converse-only" + "prompt_caching_supported": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported (base URL bedrock-runtime.{region}.amazonaws.com/openai/v1); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol; the card's runtime table is the source of this model's inference_profiles matrix (us. and global. only); Guardrails Converse-only; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html, read 2026-11-09)" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-sol", "requires_cris": true, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported (/openai/v1/chat/completions); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol; the card's runtime table is the source of this model's inference_profiles matrix (us. and global. only); Guardrails Converse-only" + "prompt_caching_supported": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported (/openai/v1/chat/completions); no in-region form; Geo id us.openai.gpt-5.6-sol, Global id global.openai.gpt-5.6-sol; the card's runtime table is the source of this model's inference_profiles matrix (us. and global. only); Guardrails Converse-only; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html, read 2026-11-09)" } } }, @@ -209,38 +214,43 @@ "model_id": "openai.gpt-5.5", "requires_cris": false, "guardrails_supported": false, - "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1 and us-east-2 only; Geo/Global not supported on mantle" + "prompt_caching_supported": false, + "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1 and us-east-2 only; Geo/Global not supported on mantle; prompt_caching_supported false: the card's bedrock-mantle feature list names no prompt caching (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, read 2026-11-09)" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": false, "guardrails_supported": false, - "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code" + "prompt_caching_supported": false, + "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code; prompt_caching_supported false: the card's bedrock-mantle feature list names no prompt caching (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, read 2026-11-09)" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": true, "guardrails_supported": true, + "prompt_caching_supported": false, "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision", "streaming"], - "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.5 (Geo) or global.openai.gpt-5.5 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): 'Response streaming' listed for runtime; 'Structured outputs (JSON Schema; Responses API)' is scoped to Responses, so structured_output_json is not carried on Converse; Guardrails Converse-only" + "evidence": "AWS model card model-card-openai-gpt-55 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.5 (Geo) or global.openai.gpt-5.5 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): 'Response streaming' listed for runtime; 'Structured outputs (JSON Schema; Responses API)' is scoped to Responses, so structured_output_json is not carried on Converse; Guardrails Converse-only; prompt_caching_supported false: the card lists 'Implicit Prompt Caching (Responses API only; automatic, no request parameters)' on bedrock-runtime and Explicit Prompt Caching as not supported (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, read 2026-11-09)" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": true, "guardrails_supported": false, + "prompt_caching_supported": true, "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision", "structured_output_json", "streaming"], - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): bedrock-runtime lists Responses supported; 'Structured outputs (JSON Schema; Responses API)' and 'Response streaming' listed for runtime, implicit prompt caching Responses-only; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): bedrock-runtime lists Responses supported; 'Structured outputs (JSON Schema; Responses API)' and 'Response streaming' listed for runtime, implicit prompt caching Responses-only; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5; prompt_caching_supported true: the card lists 'Implicit Prompt Caching (Responses API only; automatic, no request parameters)' on bedrock-runtime and Explicit Prompt Caching as not supported (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, read 2026-11-09)" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.5", "requires_cris": true, "guardrails_supported": false, + "prompt_caching_supported": false, "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision", "streaming"], - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; 'Response streaming' listed for runtime; the card scopes 'Structured outputs (JSON Schema; Responses API)' and implicit prompt caching to Responses, so structured_output_json is not carried on Chat Completions; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5" + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; 'Response streaming' listed for runtime; the card scopes 'Structured outputs (JSON Schema; Responses API)' and implicit prompt caching to Responses, so structured_output_json is not carried on Chat Completions; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.5, Global id global.openai.gpt-5.5; prompt_caching_supported false: the card lists 'Implicit Prompt Caching (Responses API only; automatic, no request parameters)' on bedrock-runtime and Explicit Prompt Caching as not supported (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html, read 2026-11-09)" } } }, @@ -281,35 +291,40 @@ "model_id": "openai.gpt-5.6-terra", "requires_cris": false, "guardrails_supported": false, - "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard" + "prompt_caching_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard; prompt_caching_supported true: the card lists implicit prompt caching on the Responses API only on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html, read 2026-11-09)" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": false, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2, us-west-2, us-gov-west-1, us-gov-east-1" + "prompt_caching_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2, us-west-2, us-gov-west-1, us-gov-east-1; prompt_caching_supported false: the card lists implicit prompt caching on the Responses API only on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html, read 2026-11-09)" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": true, "guardrails_supported": true, - "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): us. from six US/Canada source regions, in. from ap-south-1/ap-south-2, us-gov. from us-gov-east-1/us-gov-west-1 (Global not supported there), global. from every listed commercial region" + "prompt_caching_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): us. from six US/Canada source regions, in. from ap-south-1/ap-south-2, us-gov. from us-gov-east-1/us-gov-west-1 (Global not supported there), global. from every listed commercial region; prompt_caching_supported true: the card now lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime, superseding the 2026-08-21 Responses-only note (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html, read 2026-11-09)" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": true, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" + "prompt_caching_supported": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html, read 2026-11-09)" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-terra", "requires_cris": true, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" + "prompt_caching_supported": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-terra (commercial), in.openai.gpt-5.6-terra (India), us-gov.openai.gpt-5.6-terra (GovCloud); Global id global.openai.gpt-5.6-terra; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html, read 2026-11-09)" } } }, @@ -350,35 +365,40 @@ "model_id": "openai.gpt-5.6-luna", "requires_cris": false, "guardrails_supported": false, - "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard" + "prompt_caching_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-mantle at /openai/v1, in-region only; Responses supported; In-Region pricing at 1.10x OpenAI standard; prompt_caching_supported true: the card lists implicit prompt caching on the Responses API only on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html, read 2026-11-09)" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": false, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2, us-west-2, us-gov-west-1, us-gov-east-1" + "prompt_caching_supported": false, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-mantle lists Chat Completions supported at /openai/v1/chat/completions; in-region us-east-1, us-east-2, us-west-2, us-gov-west-1, us-gov-east-1; prompt_caching_supported false: the card lists implicit prompt caching on the Responses API only on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html, read 2026-11-09)" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": true, "guardrails_supported": true, - "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): us. from six US/Canada source regions, in. from ap-south-1/ap-south-2, us-gov. from us-gov-east-1/us-gov-west-1 (Global not supported there), global. from every listed commercial region" + "prompt_caching_supported": true, + "evidence": "AWS model card (2026-08-21): bedrock-runtime row with Geo (us./in.) and Global CRIS ids; APIs listed: Responses/Chat Completions/Invoke/Converse; Guardrails Converse-only, prompt caching Responses-only; Global CRIS priced at OpenAI standard (parity), In-Region/Geo at 1.10x. Profile matrix (inference_profiles) from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): us. from six US/Canada source regions, in. from ap-south-1/ap-south-2, us-gov. from us-gov-east-1/us-gov-west-1 (Global not supported there), global. from every listed commercial region; prompt_caching_supported true: the card now lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime, superseding the 2026-08-21 Responses-only note (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html, read 2026-11-09)" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": true, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" + "prompt_caching_supported": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-runtime lists Responses supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html, read 2026-11-09)" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.6-luna", "requires_cris": true, "guardrails_supported": false, - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only" + "prompt_caching_supported": true, + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; no in-region form; Geo ids us.openai.gpt-5.6-luna (commercial), in.openai.gpt-5.6-luna (India), us-gov.openai.gpt-5.6-luna (GovCloud); Global id global.openai.gpt-5.6-luna; the card's runtime table is the source of this model's inference_profiles matrix; Guardrails Converse-only; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html, read 2026-11-09)" } } }, @@ -420,39 +440,44 @@ "model_id": "openai.gpt-5.4", "requires_cris": false, "guardrails_supported": false, - "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1, us-east-2, us-west-2 and us-gov-west-1; Geo/Global not supported on mantle" + "prompt_caching_supported": false, + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-mantle supports Responses at /openai/v1/responses; in-region us-east-1, us-east-2, us-west-2 and us-gov-west-1; Geo/Global not supported on mantle; prompt_caching_supported false: the card's bedrock-mantle feature list names no prompt caching (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, read 2026-11-09)" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": false, "guardrails_supported": false, + "prompt_caching_supported": false, "capabilities": ["tool_or_function_calling", "reasoning", "streaming", "image_input_vision"], - "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): structured outputs are evidenced on the Responses API only, so structured_output_json is not carried on Mantle Chat Completions" + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): Chat Completions listed as supported on bedrock-mantle at /openai/v1/chat/completions, with sample code. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): structured outputs are evidenced on the Responses API only, so structured_output_json is not carried on Mantle Chat Completions; prompt_caching_supported false: the card's bedrock-mantle feature list names no prompt caching (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, read 2026-11-09)" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": true, "guardrails_supported": true, + "prompt_caching_supported": false, "capabilities": ["tool_or_function_calling", "reasoning", "streaming", "image_input_vision"], - "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.4 (Geo) or global.openai.gpt-5.4 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): 'Response streaming' listed for runtime; 'Structured outputs (JSON Schema; Responses API)' is scoped to Responses, so structured_output_json is not carried on Converse; Guardrails Converse-only" + "evidence": "AWS model card model-card-openai-gpt-54 (read 2026-11-06): bedrock-runtime supported with Responses, Chat Completions and Converse (Messages and Invoke unsupported); no in-region form — requests must name us.openai.gpt-5.4 (Geo) or global.openai.gpt-5.4 (Global); base URL bedrock-runtime.{region}.amazonaws.com/openai/v1 for the OpenAI-compatible APIs. Per-API capabilities from https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): 'Response streaming' listed for runtime; 'Structured outputs (JSON Schema; Responses API)' is scoped to Responses, so structured_output_json is not carried on Converse; Guardrails Converse-only; prompt_caching_supported false: the card lists 'Implicit Prompt Caching (Responses API only; automatic, no request parameters)' on bedrock-runtime and Explicit Prompt Caching as not supported (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, read 2026-11-09)" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": true, "guardrails_supported": false, + "prompt_caching_supported": true, "capabilities": ["tool_or_function_calling", "structured_output_json", "reasoning", "streaming", "image_input_vision"], - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): bedrock-runtime lists Responses supported; 'Structured outputs (JSON Schema; Responses API)' and 'Response streaming' listed for runtime, implicit prompt caching Responses-only; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): bedrock-runtime lists Responses supported; 'Structured outputs (JSON Schema; Responses API)' and 'Response streaming' listed for runtime, implicit prompt caching Responses-only; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4; prompt_caching_supported true: the card lists 'Implicit Prompt Caching (Responses API only; automatic, no request parameters)' on bedrock-runtime and Explicit Prompt Caching as not supported (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, read 2026-11-09)" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-5.4", "requires_cris": true, "guardrails_supported": false, + "prompt_caching_supported": false, "capabilities": ["tool_or_function_calling", "reasoning", "streaming", "image_input_vision"], - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; 'Response streaming' listed for runtime; the card scopes 'Structured outputs (JSON Schema; Responses API)' and implicit prompt caching to Responses, so structured_output_json is not carried on Chat Completions; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4" + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; 'Response streaming' listed for runtime; the card scopes 'Structured outputs (JSON Schema; Responses API)' and implicit prompt caching to Responses, so structured_output_json is not carried on Chat Completions; Guardrails Converse-only; no in-region form; Geo id us.openai.gpt-5.4, Global id global.openai.gpt-5.4; prompt_caching_supported false: the card lists 'Implicit Prompt Caching (Responses API only; automatic, no request parameters)' on bedrock-runtime and Explicit Prompt Caching as not supported (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html, read 2026-11-09)" } } }, @@ -511,6 +536,7 @@ "model_id": "openai.gpt-6-astra", "requires_cris": false, "guardrails_supported": false, + "prompt_caching_supported": true, "supported_regions": [ "us-east-1", "us-west-2" @@ -521,13 +547,14 @@ "tool_or_function_calling", "prompt_caching" ], - "evidence": "AWS model card 2026-09-30: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html. Responses supported at /openai/v1/responses; Standard mantle is available in us-east-1 and us-west-2. Server-side tool calling and implicit/explicit Responses caching supported; application inference profiles unsupported." + "evidence": "AWS model card 2026-09-30: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html. Responses supported at /openai/v1/responses; Standard mantle is available in us-east-1 and us-west-2. Server-side tool calling and implicit/explicit Responses caching supported; application inference profiles unsupported.; prompt_caching_supported mirrors this row's cataloged capabilities (prompt_caching present or absent; https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html, read 2026-11-09)" }, "mantle_openai_chat": { "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": false, "guardrails_supported": false, + "prompt_caching_supported": false, "supported_regions": [ "us-east-1", "us-west-2" @@ -536,13 +563,14 @@ "reasoning", "image_input_vision" ], - "evidence": "AWS model card 2026-09-30: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html. Chat Completions supported at /openai/v1/chat/completions in us-east-1 and us-west-2; prompt caching is Responses-only. Tool behavior on Chat Completions remains unprobed." + "evidence": "AWS model card 2026-09-30: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html. Chat Completions supported at /openai/v1/chat/completions in us-east-1 and us-west-2; prompt caching is Responses-only. Tool behavior on Chat Completions remains unprobed.; prompt_caching_supported mirrors this row's cataloged capabilities (prompt_caching present or absent; https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html, read 2026-11-09)" }, "runtime_converse": { "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": true, "guardrails_supported": true, + "prompt_caching_supported": false, "supported_regions": [ "us-east-1", "us-east-2", @@ -596,13 +624,14 @@ "streaming", "structured_output_json" ], - "evidence": "AWS model card, runtime JSON Schema reverified 2026-10-03: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html. Runtime APIs: Responses, Chat Completions, Converse; Invoke and Messages unsupported. US Geo/Global CRIS only, no in-region or EU/India Geo form. Guardrails and application inference profiles are Converse-only. Non-streaming JSON Schema outputs supported: Converse requires outputConfig.textFormat.structure.jsonSchema and additionalModelRequestFields.text.format.strict=true. Streaming structured output and server-side tool use are not evidenced. Runtime TPM accounting uses 10x output-token burndown." + "evidence": "AWS model card, runtime JSON Schema reverified 2026-10-03: https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html. Runtime APIs: Responses, Chat Completions, Converse; Invoke and Messages unsupported. US Geo/Global CRIS only, no in-region or EU/India Geo form. Guardrails and application inference profiles are Converse-only. Non-streaming JSON Schema outputs supported: Converse requires outputConfig.textFormat.structure.jsonSchema and additionalModelRequestFields.text.format.strict=true. Streaming structured output and server-side tool use are not evidenced. Runtime TPM accounting uses 10x output-token burndown.; prompt_caching_supported mirrors this row's cataloged capabilities (prompt_caching present or absent; https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html, read 2026-11-09)" }, "runtime_openai_responses": { "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": true, "guardrails_supported": false, + "prompt_caching_supported": true, "supported_regions": [ "us-east-1", "us-east-2", @@ -657,13 +686,14 @@ "structured_output_json", "prompt_caching" ], - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html (read 2026-11-07): bedrock-runtime lists Responses supported; runtime features: response streaming, structured outputs (JSON Schema), implicit and explicit prompt caching on Responses, reasoning summaries; application inference profiles are InvokeModel/Converse-only and server-side tool use is unsupported; no in-region form; Geo id us.openai.gpt-6-astra, Global id global.openai.gpt-6-astra" + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html (read 2026-11-07): bedrock-runtime lists Responses supported; runtime features: response streaming, structured outputs (JSON Schema), implicit and explicit prompt caching on Responses, reasoning summaries; application inference profiles are InvokeModel/Converse-only and server-side tool use is unsupported; no in-region form; Geo id us.openai.gpt-6-astra, Global id global.openai.gpt-6-astra; prompt_caching_supported mirrors this row's cataloged capabilities (prompt_caching present or absent; https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html, read 2026-11-09)" }, "runtime_openai_chat": { "available": true, "model_id": "openai.gpt-6-astra", "requires_cris": true, "guardrails_supported": false, + "prompt_caching_supported": true, "supported_regions": [ "us-east-1", "us-east-2", @@ -718,7 +748,7 @@ "structured_output_json", "prompt_caching" ], - "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; runtime features: response streaming, structured outputs (JSON Schema), implicit and explicit prompt caching on Chat Completions; application inference profiles are InvokeModel/Converse-only and server-side tool use is unsupported; no in-region form; Geo id us.openai.gpt-6-astra, Global id global.openai.gpt-6-astra" + "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html (read 2026-11-07): bedrock-runtime lists Chat Completions supported; runtime features: response streaming, structured outputs (JSON Schema), implicit and explicit prompt caching on Chat Completions; application inference profiles are InvokeModel/Converse-only and server-side tool use is unsupported; no in-region form; Geo id us.openai.gpt-6-astra, Global id global.openai.gpt-6-astra; prompt_caching_supported mirrors this row's cataloged capabilities (prompt_caching present or absent; https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html, read 2026-11-09)" } } }, @@ -742,11 +772,11 @@ "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision"], "capability_source": "AWS GPT-6 Sol model card, retrieved 2026-10-10: text and image input, tool calling, 1.05M context and 128K output.", "paths": { - "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, - "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported. (Guardrails: Responses/Chat Completions/InvokeModel/Converse per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html; card read 2026-11-08)"}, - "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"}, - "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol"} + "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "guardrails_supported": false, "prompt_caching_supported": true, "evidence": "AWS model card: Mantle Responses in us-east-1.; prompt_caching_supported true: the card lists implicit prompt caching on Responses and Chat Completions on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html, read 2026-11-09)"}, + "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": false, "guardrails_supported": false, "prompt_caching_supported": true, "evidence": "AWS model card: Mantle Chat Completions supported.; prompt_caching_supported true: the card lists implicit prompt caching on Responses and Chat Completions on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html, read 2026-11-09)"}, + "runtime_converse": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported. (Guardrails: Responses/Chat Completions/InvokeModel/Converse per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html; card read 2026-11-08); prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html, read 2026-11-09)"}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html, read 2026-11-09)"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-sol", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-sol, Global id global.openai.gpt-6-sol; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-sol.html, read 2026-11-09)"} } }, "openai_gpt_6_luna": { @@ -769,11 +799,11 @@ "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision"], "capability_source": "AWS GPT-6 Luna model card, retrieved 2026-10-10: text and image input, tool calling, 1.05M context and 128K output.", "paths": { - "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Responses in us-east-1."}, - "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "guardrails_supported": false, "evidence": "AWS model card: Mantle Chat Completions supported."}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke. (Guardrails: Responses/Chat Completions/InvokeModel/Converse per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html; card read 2026-11-08)"}, - "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"}, - "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna"} + "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "guardrails_supported": false, "prompt_caching_supported": true, "evidence": "AWS model card: Mantle Responses in us-east-1.; prompt_caching_supported true: the card lists implicit prompt caching on Responses and Chat Completions on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html, read 2026-11-09)"}, + "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": false, "guardrails_supported": false, "prompt_caching_supported": true, "evidence": "AWS model card: Mantle Chat Completions supported.; prompt_caching_supported true: the card lists implicit prompt caching on Responses and Chat Completions on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html, read 2026-11-09)"}, + "runtime_converse": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke. (Guardrails: Responses/Chat Completions/InvokeModel/Converse per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html; card read 2026-11-08); prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html, read 2026-11-09)"}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html, read 2026-11-09)"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6-luna", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails on Responses/Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6-luna, Global id global.openai.gpt-6-luna; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html, read 2026-11-09)"} } }, "openai_gpt_6_1_sol": { @@ -796,11 +826,11 @@ "capabilities": ["reasoning", "tool_or_function_calling", "image_input_vision"], "capability_source": "AWS GPT-6.1 Sol model card, retrieved 2026-10-10: text and image input, tool calling, 1.05M context and 131072 output.", "paths": { - "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "guardrails_supported": false, "server_side_tools_supported": false, "evidence": "AWS model card: Mantle Responses in us-east-1. https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): Server-side system tools listed as Not Supported on bedrock-mantle"}, - "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "guardrails_supported": false, "server_side_tools_supported": false, "evidence": "AWS model card: Mantle Chat Completions supported. https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): Server-side system tools listed as Not Supported on bedrock-mantle"}, - "runtime_converse": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported. (Guardrails: Chat Completions/InvokeModel/Converse, not Responses, per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html; card read 2026-11-08)"}, - "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": false, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails are listed for Chat Completions/InvokeModel/Converse but not Responses, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"}, - "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails listed on Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol"} + "mantle_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "guardrails_supported": false, "prompt_caching_supported": true, "server_side_tools_supported": false, "evidence": "AWS model card: Mantle Responses in us-east-1. https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): Server-side system tools listed as Not Supported on bedrock-mantle; prompt_caching_supported true: the card lists implicit prompt caching on Responses and Chat Completions on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html, read 2026-11-09)"}, + "mantle_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": false, "guardrails_supported": false, "prompt_caching_supported": true, "server_side_tools_supported": false, "evidence": "AWS model card: Mantle Chat Completions supported. https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): Server-side system tools listed as Not Supported on bedrock-mantle; prompt_caching_supported true: the card lists implicit prompt caching on Responses and Chat Completions on bedrock-mantle (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html, read 2026-11-09)"}, + "runtime_converse": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "AWS model card: US and Global CRIS support Responses, Chat Completions, Converse, and Invoke; EU/AU Geo profiles are unsupported. (Guardrails: Chat Completions/InvokeModel/Converse, not Responses, per https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html; card read 2026-11-08); prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html, read 2026-11-09)"}, + "runtime_openai_responses": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": false, "prompt_caching_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): bedrock-runtime lists Responses supported; Guardrails are listed for Chat Completions/InvokeModel/Converse but not Responses, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html, read 2026-11-09)"}, + "runtime_openai_chat": {"available": true, "model_id": "openai.gpt-6.1-sol", "requires_cris": true, "guardrails_supported": true, "prompt_caching_supported": true, "evidence": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html (read 2026-11-08): bedrock-runtime lists Chat Completions supported; Guardrails listed on Chat Completions/InvokeModel/Converse, application inference profiles InvokeModel/Converse-only; no in-region form; Geo id us.openai.gpt-6.1-sol, Global id global.openai.gpt-6.1-sol; prompt_caching_supported true: the card lists implicit prompt caching on Responses, Chat Completions, InvokeModel and Converse on bedrock-runtime (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html, read 2026-11-09)"} } }, "anthropic_claude_sonnet_5": { From dd26c7509b50661e5931a8ab17ca9d1a83670442 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 9 Nov 2026 14:49:28 -0800 Subject: [PATCH 40/41] fix(aws-startup-advisor): keep Responses-only features and Web Search grants when another selector is primary A Chat path no longer hides web_search, background inference, or server-side tools behind projects. Claude Converse rows carry prompt-caching evidence, and IAM receives every confirmed selector. Co-authored-by: Cursor --- .../agents/llm2bedrock-report-generator.md | 11 +- .../phases/model-recommend/model-recommend.md | 17 +-- .../vendored/ai/models/openai-bedrock.json | 10 +- .../scripts/openai_model_recommendation.py | 54 ++++++-- .../test_openai_model_recommendation.py | 116 +++++++++++++----- .../references/phases/design/design-ai.md | 11 +- .../phases/generate/generate-artifacts-ai.md | 6 +- .../references/shared/schema-design-aws-ai.md | 2 +- .../vendored/ai/ai-openai-to-bedrock.md | 2 +- .../vendored/ai/schema-design-aws-ai.md | 4 +- .../references/phases/design/design-ai.md | 4 +- .../phases/generate/generate-artifacts-ai.md | 2 +- .../references/shared/schema-design-aws-ai.md | 2 +- .../vendored/ai/ai-openai-to-bedrock.md | 2 +- .../vendored/ai/schema-design-aws-ai.md | 4 +- .../skills/shared/ai/ai-openai-to-bedrock.md | 2 +- .../shared/ai/models/openai-bedrock.json | 10 +- .../skills/shared/ai/schema-design-aws-ai.md | 4 +- .../test_runtime_default_review_round3.py | 6 +- 19 files changed, 188 insertions(+), 81 deletions(-) diff --git a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md index 613df593c..e8ae939b4 100644 --- a/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md +++ b/plugins/aws-startup-advisor/agents/llm2bedrock-report-generator.md @@ -630,9 +630,12 @@ plan, which has no Responses row on either endpoint — report that verbatim (th recorded ops for such a source) and do not add the actions by hand. When the field is absent or `[]`, omit the flag; the policy then carries no `GetInvoke` / `CancelInvoke` / `DeleteInvoke`. -**Web Search needs its own grants.** When any `bedrock_models[]` row has `mantle_selector: web_search`, -pass `--mantle-selector web_search` (join every `mantle_*` row's `mantle_selector` with commas; the other -selectors add nothing) and the helper adds a `BedrockWebSearch` statement: `bedrock-websearch:InvokeSearch` +**Web Search needs its own grants.** Pass every confirmed selector, not only each row's primary +`mantle_selector`. Union `preferences.json` → `ai_constraints.mantle_requirements.value[]` entries whose +`source` is `detected` or `user` with every `bedrock_models[].mantle_selector`. When that set contains +`web_search` — including when the row's primary selector is `background_inference` and Web Search is only +named in `rationale` — pass `--mantle-selector web_search` joined with the other confirmed selectors (those +other selectors add no Web Search actions) and the helper adds a `BedrockWebSearch` statement: `bedrock-websearch:InvokeSearch` and `bedrock-websearch:InvokeFetch` on resource `*` with an `aws:RequestedRegion` condition — the three `bedrock-websearch` actions all take `*` and that global key is the service's only condition key (, read 2026-11-09). The @@ -952,7 +955,7 @@ line `*Monthly estimate unavailable — sample has no time-span information.*` prove retrieval is authorized. - - + - **Tool-level permission check** (a plain model ping passing does not prove the Web Search grant): under the attached identity run `client.responses.create(model="", input="", tools=[{"type": "web_search", "external_web_access": False}])` diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md index a4949f9b9..3d4c6a061 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/model-recommend/model-recommend.md @@ -150,14 +150,17 @@ Converse — the bedrock-runtime Responses and Chat Completions APIs reject it w (`https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html`, read 2026-11-08) — so with required continuity it is a `decision_required` whose OpenAI-API option carries no invocation id; system, Geo and Global profiles are unaffected. Every selector is -validated against the final provider and API before a path is chosen. `background_inference` and -`web_search` are Responses API features on `bedrock-mantle` only +validated against the final provider and API before a path is chosen, and a retained +`critical_features` entry is checked the same way even when another selector is the primary +reason. `background_inference`, `web_search`, and `server_side_tools` are Responses API features +on `bedrock-mantle` (`https://docs.aws.amazon.com/bedrock/latest/userguide/inference-responses-api.html`, -`https://docs.aws.amazon.com/bedrock/latest/userguide/web-search.html`, read 2026-11-09), so a -Chat Completions source matched by either selector is a `decision_required` -(`selector_api_incompatible`: `mantle_openai_responses` carrying the selector versus -`runtime_openai_chat` without it) unless `api_continuity: not_required` allows the reshape; -`mantle_openai_chat` is never recorded or offered for a Responses-only selector. A selector whose +`https://docs.aws.amazon.com/bedrock/latest/userguide/web-search.html`, +`https://docs.aws.amazon.com/bedrock/latest/userguide/tool-use-server-side.html`, read 2026-11-09), +so a Chat Completions path that would carry one of them is a `decision_required` +(`selector_api_incompatible`: `mantle_openai_responses` versus `runtime_openai_chat` without it) +unless `api_continuity: not_required` allows the reshape; `mantle_openai_chat` is never recorded +or offered for a Responses-only feature. Client function tools stay on Chat Completions. A selector whose provider does not match the workload is a `selector_provider_mismatch` decision whose options both stay on `bedrock-runtime`: `workspaces` (`anthropic-workspace-id`, `https://docs.aws.amazon.com/bedrock/latest/userguide/workspaces.html`) on an OpenAI SDK diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json index 1ba299cdb..933c76e11 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/vendored/ai/models/openai-bedrock.json @@ -853,7 +853,8 @@ "model_id": "anthropic.claude-sonnet-5", "requires_cris": true, "guardrails_supported": true, - "evidence": "AWS model card (2026-07-21): Converse on bedrock-runtime checked; Geo (us/eu/au) and Global CRIS supported" + "prompt_caching_supported": true, + "evidence": "AWS model card (2026-07-21): Converse on bedrock-runtime checked; Geo (us/eu/au) and Global CRIS supported; prompt_caching_supported true: Claude explicit cache checkpoints (cachePoint) are supported on Converse (https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html, read 2026-11-09)" } } }, @@ -876,7 +877,8 @@ "model_id": "anthropic.claude-haiku-4-5-20251001-v1:0", "requires_cris": true, "guardrails_supported": true, - "evidence": "anthropic-bedrock.json catalog (verified 2026-09-24) runtime_converse entry (requires CRIS profile)" + "prompt_caching_supported": true, + "evidence": "anthropic-bedrock.json catalog (verified 2026-09-24) runtime_converse entry (requires CRIS profile); prompt_caching_supported true: Claude explicit cache checkpoints (cachePoint) are supported on Converse (https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html, read 2026-11-09)" } } }, @@ -970,7 +972,9 @@ "available": true, "model_id": "anthropic.claude-opus-5-5", "requires_cris": true, - "guardrails_supported": true + "guardrails_supported": true, + "prompt_caching_supported": true, + "evidence": "prompt_caching_supported true: Claude explicit cache checkpoints (cachePoint) are supported on Converse (https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html, read 2026-11-09)" } }, "generation": "bedrock_native", diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py index 371f84b2e..70443bcdc 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/openai_model_recommendation.py @@ -320,6 +320,22 @@ def _mantle_selector(workload, feature_assessment): return None +def _chat_responses_blocker(selector, requirements, detected): + """The Responses-only feature a Chat Mantle path would drop. + + The primary selector is not the only input: a projects or server_side_tools + selector must not hide critical or detected web_search / background_inference. + """ + needed = set(requirements.get("critical_features") or []) | set(detected or []) + if selector == "web_search" or "web_search" in needed: + return "web_search" + if selector == "background_inference" or "background_inference" in needed: + return "background_inference" + if selector == "server_side_tools": + return "server_side_tools" + return None + + def _openai_candidate_order(catalog, source): """[same model] + OpenAI-generation models, same family as the source's GPT major first, then version descending, then catalog order. Sources with no parseable @@ -367,7 +383,10 @@ def _hard_profile_requirement(requirements): # (inference-responses-api.html) and the built-in Web Search tool (web-search.html), # both read 2026-11-09. Chat Completions has neither, so a Chat source matched by # one of these is an explicit reshape decision, never a same-API Mantle Chat row. -_RESPONSES_ONLY_SELECTORS = frozenset({"background_inference", "web_search"}) +# server_side_tools joins this set: the documented Lambda/MCP server-tool flow uses +# Responses items (tool-use-server-side.html, read 2026-11-09). Client function tools +# are a capability, not this selector, and stay on Chat Completions. +_RESPONSES_ONLY_SELECTORS = frozenset({"background_inference", "web_search", "server_side_tools"}) def _catalog_model_for_path(catalog, path, detected_features=None, requirements=None, @@ -1027,7 +1046,7 @@ def _compatibility(detected_features, requirements, path, target_model=None, pat unsupported = sorted(text_features - catalog_caps) if target_model else [] hosted = detected.intersection(_HOSTED_TOOL_IMPACTS) portable_features = detected.intersection(_PORTABLE_FEATURES) - if (path or "").startswith("mantle_") and "web_search" in hosted: + if (path or "").startswith("mantle_") and path != "mantle_openai_chat" and "web_search" in hosted: # Web Search is a Mantle server-side tool: on a Mantle path it ports. hosted.discard("web_search") portable_features.add("web_search") @@ -1593,18 +1612,19 @@ def _preferred_path_conflict(mantle_selector, message): ) path = preferred - # Background inference and Web Search are Responses API features on - # bedrock-mantle; a Chat Completions Mantle path cannot carry either, so the - # reshape is offered as a decision rather than selected silently. - if selector in _RESPONSES_ONLY_SELECTORS and path == "mantle_openai_chat": - if selector == "web_search": + # A Chat Completions Mantle path cannot carry a Responses-only feature. Check the + # retained features themselves, not only the primary selector: projects or + # server_side_tools must not hide a required web_search or background_inference. + blocker = _chat_responses_blocker(selector, requirements, detected) + if path == "mantle_openai_chat" and blocker: + if blocker == "web_search": message = ( "Web Search is a server-side tool served on the bedrock-mantle Responses API " "(https://docs.aws.amazon.com/bedrock/latest/userguide/web-search.html, " "read 2026-11-09); Chat Completions has no `web_search` tool." ) dropped = "Web Search" - else: + elif blocker == "background_inference": message = ( "Background inference is a Responses API feature on bedrock-mantle; Chat " "Completions has no background mode " @@ -1612,8 +1632,22 @@ def _preferred_path_conflict(mantle_selector, message): "read 2026-11-08)." ) dropped = "background inference" + else: + message = ( + "Server-side tools (Lambda-backed and MCP tools) use Responses request and " + "response items on bedrock-mantle " + "(https://docs.aws.amazon.com/bedrock/latest/userguide/tool-use-server-side.html, " + "read 2026-11-09); Chat Completions has no server-side tool items. Client " + "function tools stay on Chat Completions." + ) + dropped = "server-side tools" + if selector and selector != blocker: + message += ( + f" mantle_selector {selector} does not satisfy this retained requirement." + ) + option_selector = blocker if blocker in MANTLE_SELECTORS else selector return _unresolved( - f"mantle_selector {selector} conflicts with the Chat Completions API.", + f"{blocker} conflicts with the Chat Completions API.", [], _finding( "selector_api_incompatible", "[BLOCKS]", message, @@ -1623,7 +1657,7 @@ def _preferred_path_conflict(mantle_selector, message): path, decision_options=[ option for option in _decision_options( - catalog, workload, region, selector, openai_path="mantle_openai_responses" + catalog, workload, region, option_selector, openai_path="mantle_openai_responses" ) if option["api_path"] == "mantle_openai_responses" ] + _runtime_option("runtime_openai_chat"), diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py index 9677ea85e..66a78cf66 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_openai_model_recommendation.py @@ -733,6 +733,68 @@ def test_chat_source_with_web_search_plus_logging_never_offers_a_mantle_chat_opt assert _option(rec, "mantle_openai_responses")["mantle_selector"] == "web_search" +@pytest.mark.parametrize("selector", ["projects", "server_side_tools"]) +def test_required_chat_plus_another_selector_still_validates_critical_web_search(selector): + rec = _recommend( + _workload(source={"model_ids": ["gpt-5.5"], "api_surface": "chat_completions"}, + requirements={"api_continuity": "required", "mantle_selector": selector, + "data_residency": "global_allowed", + "critical_features": ["web_search"]}), + region="us-east-1", + ) + assert rec["decision_status"] == "decision_required" + assert rec["api_path"] is None + assert "selector_api_incompatible" in _codes(rec["blocks"]) + assert "web-search.html" in rec["blocks"][0]["message"] + assert f"mantle_selector {selector}" in rec["blocks"][0]["message"] + assert {o["api_path"] for o in rec["decision_options"]} == { + "mantle_openai_responses", "runtime_openai_chat", + } + assert _option(rec, "mantle_openai_responses")["mantle_selector"] == "web_search" + assert not any(o["api_path"] == "mantle_openai_chat" for o in rec["decision_options"]) + + +def test_server_side_tools_on_required_chat_is_a_responses_decision(): + rec = _recommend( + _workload(source={"model_ids": ["gpt-5.6-terra"], "api_surface": "chat_completions"}, + requirements={"api_continuity": "required", + "mantle_selector": "server_side_tools"}), + region="us-east-1", + ) + assert rec["decision_status"] == "decision_required" + assert "tool-use-server-side.html" in rec["blocks"][0]["message"] + assert "Client function tools stay on Chat Completions" in rec["blocks"][0]["message"] + assert {o["api_path"] for o in rec["decision_options"]} == { + "mantle_openai_responses", "runtime_openai_chat", + } + + +def test_client_function_tools_on_required_chat_stay_on_chat(): + rec = _recommend( + _workload(source={"model_ids": ["gpt-5.6-terra"], "api_surface": "chat_completions"}, + requirements={"api_continuity": "required"}, + detected_features=["tool_or_function_calling"]), + region="us-east-1", + ) + assert rec["decision_status"] == "recommended" + assert rec["api_path"] == "runtime_openai_chat" + + +def test_gpt4o_guardrails_and_prompt_caching_keeps_the_claude_converse_fallback(): + rec = _recommend( + _workload(source={"model_ids": ["gpt-4o"], "api_surface": "chat_completions"}, + requirements={"governance": ["guardrails"], + "api_continuity": "not_required", + "data_residency": "global_allowed", + "critical_features": ["prompt_caching"]}), + region="us-east-1", + ) + assert rec["decision_status"] == "recommended" + assert rec["primary_model"] == "anthropic.claude-sonnet-5" + assert rec["api_path"] == "runtime_converse" + assert "prompt_caching" in rec["compatibility"]["native"] + + def test_workspaces_selector_on_an_openai_source_is_a_provider_mismatch_decision(): out = _full_recommend(_workload(requirements={"mantle_selector": "workspaces"})) jsonschema.validate(out, _OUT_SCHEMA) @@ -757,7 +819,6 @@ def test_workspaces_selector_on_an_openai_source_is_a_provider_mismatch_decision @pytest.mark.parametrize(("surface", "selector", "path"), [ ("responses", "server_side_tools", "mantle_openai_responses"), - ("chat_completions", "server_side_tools", "mantle_openai_chat"), # Web Search is Responses-only: the Chat source reshapes once continuity is free. ("chat_completions", "web_search", "mantle_openai_responses"), ]) @@ -997,7 +1058,7 @@ def test_active_catalog_contains_gpt6_profile_metadata_and_current_provenance(): @pytest.mark.parametrize(("source_id", "target_id"), _GPT6_MODELS) @pytest.mark.parametrize(("selector", "path"), [ (None, "runtime_openai_chat"), - ("server_side_tools", "mantle_openai_chat"), + ("projects", "mantle_openai_chat"), ]) def test_gpt6_chat_completions_preserves_chat_surface(source_id, target_id, selector, path): requirements = {"api_continuity": "required"} @@ -1010,13 +1071,7 @@ def test_gpt6_chat_completions_preserves_chat_surface(source_id, target_id, sele ), region="us-east-1", ) - if selector and source_id == "gpt-6.1-sol": - # GPT-6.1 Sol lists server-side system tools as unsupported on bedrock-mantle - # (card read 2026-11-08): same-family fallback on the same Chat surface. - assert rec["primary_model"] == "openai.gpt-6-astra" - assert rec["mantle_selector"] == selector - else: - assert rec["primary_model"] == target_id + assert rec["primary_model"] == target_id assert rec["api_path"] == path assert "chat_completions_to_responses_required" not in _codes(rec["blocks"]) @@ -1063,7 +1118,7 @@ def test_gpt6_chat_with_selector_in_us_west_2_falls_back_to_same_family_astra(so rec = _recommend( _workload( source={"model_ids": [source_id], "api_surface": "chat_completions"}, - requirements={"api_continuity": "required", "mantle_selector": "server_side_tools"}, + requirements={"api_continuity": "required", "mantle_selector": "projects"}, ), region="us-west-2", ) @@ -1071,7 +1126,7 @@ def test_gpt6_chat_with_selector_in_us_west_2_falls_back_to_same_family_astra(so assert rec["api_path"] == "mantle_openai_chat" assert rec["primary_model"] == "openai.gpt-6-astra" assert rec["primary_model"] != "openai.gpt-5.4" - assert rec["mantle_selector"] == "server_side_tools" + assert rec["mantle_selector"] == "projects" assert "fallback" in rec["rationale"][0] @@ -1083,7 +1138,7 @@ def test_gpt6_chat_with_selector_in_us_east_2_prefers_newest_gpt5_over_gpt54(sou rec = _recommend( _workload( source={"model_ids": [source_id], "api_surface": "chat_completions"}, - requirements={"api_continuity": "required", "mantle_selector": "server_side_tools"}, + requirements={"api_continuity": "required", "mantle_selector": "projects"}, ), region="us-east-2", ) @@ -1139,7 +1194,7 @@ def test_gpt56_sol_responses_in_us_west_2_stays_on_itself_or_falls_to_newest_ter def test_gpt6_chat_outside_every_mantle_footprint_returns_blocking_decision(): workload = _workload( source={"model_ids": ["gpt-6-sol"], "api_surface": "chat_completions"}, - requirements={"api_continuity": "required", "mantle_selector": "server_side_tools"}, + requirements={"api_continuity": "required", "mantle_selector": "projects"}, ) out = _full_recommend(workload, region="eu-west-1") jsonschema.validate(out, _OUT_SCHEMA) @@ -1173,7 +1228,7 @@ def test_gpt6_chat_conflict_option_names_chat_completions_surface(): source={"model_ids": ["gpt-6-sol"], "api_surface": "chat_completions"}, requirements={ "api_continuity": "required", - "mantle_selector": "server_side_tools", + "mantle_selector": "projects", "governance": ["guardrails"], }, ), @@ -1185,14 +1240,14 @@ def test_gpt6_chat_conflict_option_names_chat_completions_surface(): if option["api_path"] == "mantle_openai_chat" ) assert "Chat Completions surface" in option["reason"] - assert "server_side_tools" in option["reason"] + assert "projects" in option["reason"] assert "Responses surface" not in option["reason"] - assert option["mantle_selector"] == "server_side_tools" + assert option["mantle_selector"] == "projects" @pytest.mark.parametrize(("selector", "path"), [ (None, "runtime_openai_chat"), - ("server_side_tools", "mantle_openai_chat"), + ("projects", "mantle_openai_chat"), ]) def test_gpt6_chat_findings_preserve_chat_request_shapes(selector, path): requirements = {"api_continuity": "required", "uses_n": True, "uses_hosted_state": True} @@ -1222,7 +1277,7 @@ def test_gpt6_chat_findings_preserve_chat_request_shapes(selector, path): (None, "runtime_openai_chat", {"multi_model_converse": True}, "runtime_converse"), # GPT-6 Sol lists Guardrails on Chat Completions (card read 2026-11-08), so the # runtime side of the selector-vs-governance decision keeps the Chat API. - ("server_side_tools", "mantle_openai_chat", {"governance": ["guardrails"]}, + ("projects", "mantle_openai_chat", {"governance": ["guardrails"]}, "runtime_openai_chat"), ]) def test_gpt6_chat_conflict_outside_mantle_region_is_schema_valid( @@ -1920,15 +1975,16 @@ def test_astra_keeps_source_model_and_api_on_runtime(surface, path, region): ]) @pytest.mark.parametrize("region", ["us-east-1", "us-west-2"]) def test_astra_keeps_source_model_and_api_on_mantle_with_selector(surface, path, region): + selector = "server_side_tools" if surface == "responses" else "projects" rec = oai.recommend_openai_workload( _workload(source={"model_ids": ["gpt-6-astra"], "api_surface": surface}, - requirements={"mantle_selector": "server_side_tools"}), + requirements={"mantle_selector": selector}), region, OPENAI_CATALOG, ) assert rec["decision_status"] == "recommended" assert rec["primary_model"] == "openai.gpt-6-astra" assert rec["api_path"] == path - assert rec["mantle_selector"] == "server_side_tools" + assert rec["mantle_selector"] == selector assert "chat_completions_to_responses_required" not in _codes(rec["blocks"]) assert "sampling_params_unverified" in _codes(rec["tuning"]) @@ -1938,7 +1994,7 @@ def test_astra_keeps_source_model_and_api_on_mantle_with_selector(surface, path, def test_astra_mantle_outside_supported_regions_requires_decision(region, surface): rec = oai.recommend_openai_workload( _workload(source={"model_ids": ["gpt-6-astra"], "api_surface": surface}, - requirements={"mantle_selector": "server_side_tools"}), + requirements={"mantle_selector": "projects" if surface == "chat_completions" else "server_side_tools"}), region, OPENAI_CATALOG, ) assert rec["decision_status"] == "decision_required" @@ -1970,7 +2026,9 @@ def test_astra_mantle_outside_supported_regions_requires_decision(region, surfac def test_astra_explicit_api_preference_reaches_validated_output(surface, preferred, expected, delta): requirements = {"preferred_api_path": preferred, "data_residency": "global_allowed"} if preferred.startswith("mantle_"): - requirements["mantle_selector"] = "server_side_tools" + requirements["mantle_selector"] = ( + "projects" if preferred == "mantle_openai_chat" else "server_side_tools" + ) data = { "schema_version": 2, "region": "us-west-2", "primary_unit": "chat-svc", "workloads": [_workload( @@ -1985,7 +2043,7 @@ def test_astra_explicit_api_preference_reaches_validated_output(surface, preferr assert rec["decision_status"] == "recommended" assert rec["api_path"] == expected assert rec["primary_model"] == "openai.gpt-6-astra" - assert rec["mantle_selector"] == ("server_side_tools" if expected.startswith("mantle_") else None) + assert rec["mantle_selector"] == (requirements.get("mantle_selector") if expected.startswith("mantle_") else None) if expected.startswith("runtime_"): assert rec["invocation_model_id"] == "global.openai.gpt-6-astra" if delta: @@ -2036,10 +2094,10 @@ def test_mantle_preference_without_selector_fails_closed_for_every_source(prefer @pytest.mark.parametrize("surface,extra,expected", [ ("chat_completions", {}, "runtime_openai_chat"), ("responses", {}, "runtime_openai_responses"), - ("chat_completions", {"mantle_selector": "server_side_tools"}, None), + ("chat_completions", {"mantle_selector": "projects"}, None), ("responses", {"mantle_selector": "server_side_tools"}, "mantle_openai_responses"), ("responses", {"preferred_api_path": "mantle_openai_chat", - "mantle_selector": "server_side_tools"}, None), + "mantle_selector": "projects"}, None), ("chat_completions", {"preferred_api_path": "mantle_openai_responses", "mantle_selector": "server_side_tools"}, "mantle_openai_responses"), ]) @@ -2151,18 +2209,18 @@ def test_astra_decision_options_ignore_an_explicitly_absent_cache_signal(): workload = _workload( source={"model_ids": ["gpt-6-astra"], "api_surface": "chat_completions"}, requirements={"preferred_api_path": "mantle_openai_chat", - "mantle_selector": "server_side_tools"}, + "mantle_selector": "projects"}, detected_features=["prompt_caching"], feature_status={"prompt_caching": "absent"}, ) - options = oai._decision_options(OPENAI_CATALOG, workload, "us-west-2", "server_side_tools") + options = oai._decision_options(OPENAI_CATALOG, workload, "us-west-2", "projects") assert options[0]["api_path"] == "mantle_openai_chat" - assert options[0]["mantle_selector"] == "server_side_tools" + assert options[0]["mantle_selector"] == "projects" @pytest.mark.parametrize("extra,path", [ ({}, "runtime_openai_chat"), - ({"mantle_selector": "server_side_tools"}, "mantle_openai_chat"), + ({"mantle_selector": "projects"}, "mantle_openai_chat"), ]) @pytest.mark.parametrize("signals", [ {"requirements": {"uses_n": True}}, diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md index cbe7e883d..9f9adb738 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/design/design-ai.md @@ -112,16 +112,13 @@ Per workload (`workloads[]` from `preferences.json`, else the profile; `*` appli 1. Match `value[]` entries whose `workload_id` is the workload's id or `*`, whose `selector` is one of `background_inference | server_side_tools | web_search | projects | workspaces | in_region_processing`, and whose `source` is `detected` or `user`. `none` matches nothing. -1b. **Validate the matched selector against the API it requires.** `web_search` and - `background_inference` require the Responses API (`web-search.html`; - `inference-responses-api.html` — read 2026-11-09); `workspaces` requires the Anthropic Messages +1b. **Validate the matched selector against the API it requires.** `web_search`, `background_inference`, and `server_side_tools` require the Responses API (`web-search.html`; `inference-responses-api.html`; `tool-use-server-side.html` — read 2026-11-09); `workspaces` requires the Anthropic Messages API (`anthropic-workspace-id`); `projects` requires an OpenAI-compatible API (`OpenAI-Project`). A selector whose provider does not match the workload's source API — `workspaces` on an OpenAI SDK workload, `projects` on an Anthropic SDK workload — is a mismatch: the row keeps `bedrock-runtime`, records `endpoint_assumption`, discloses the mismatch in `rationale` and `capability_gaps[]`, and Clarify is re-run; never invent a Mantle path. A Chat Completions - source (or an Anthropic Messages source) matched by a Responses-only selector (`web_search`, - `background_inference`) is an **explicit API-continuity decision** recorded in the plan and the + source (or an Anthropic Messages source) matched by a Responses-only selector (`web_search`, `background_inference`, `server_side_tools`) is an **explicit API-continuity decision** recorded in the plan and the Present Summary, never a silent same-API Mantle row: (i) keep the source API on `bedrock-runtime` and drop the requirement — the row stays `runtime_openai_cris` / `runtime_messages` with `endpoint_assumption` and a `capability_gaps[]` entry naming the dropped @@ -130,7 +127,9 @@ Per workload (`workloads[]` from `preferences.json`, else the profile; `*` appli `mantle_openai_chat` / `mantle_messages` is never recorded for a Responses-only selector. 2. **Match that passes 1b → Mantle path for the source API** (`mantle_openai_responses` / `mantle_openai_chat` / `mantle_messages`) with `mantle_selector` = the matched selector (several → first in the order - above, name the rest in `rationale`) and `mantle_requirement` copied from the entry: + above, name the rest in `rationale`; Generate and the report pass every matched selector to + `iam_policy.py --mantle-selector`, because a grant such as Web Search is not implied by the + primary selector) and `mantle_requirement` copied from the entry: `{ workload_id, selector, source, evidence? (when detected), region? (iff in_region_processing) }`. `in_region_processing` still obeys the CRIS-only gate in `references/vendored/ai/ai-openai-to-bedrock.md`: Mantle only when `bedrock-runtime` serves the diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md index 9f54a00af..c59a4f128 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -129,8 +129,10 @@ is added and the helper does not refuse. `setup_bedrock.sh`'s verification then response it just created (`r = client.responses.create(...)`, then `client.responses.retrieve(r.id)`): a 403 on the second call means the grant is missing even though creation passed. -**Web Search grants.** When a `bedrock_models[]` row has `mantle_selector: web_search`, pass -`iam_policy.py --mantle-selector web_search` and the policy gains a `BedrockWebSearch` statement: +**Web Search grants.** When `web_search` is a confirmed mantle requirement — the row's +`mantle_selector`, or any other `detected`/`user` entry in `ai_constraints.mantle_requirements`, +including when the primary selector is `background_inference` — pass +`iam_policy.py --mantle-selector web_search` (joined with the other confirmed selectors) and the policy gains a `BedrockWebSearch` statement: `bedrock-websearch:InvokeSearch` and `bedrock-websearch:InvokeFetch` on resource `*` with an `aws:RequestedRegion` condition (all three `bedrock-websearch` actions take `*`; that is the service's only condition key — , diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md index e8352ef64..e9fc6f106 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/shared/schema-design-aws-ai.md @@ -177,7 +177,7 @@ web_search | projects | workspaces | in_region_processing` and whose `source` is comes from the model catalog with `source: "catalog"`, never from Clarify. A matching entry on a workload whose source API has no Mantle path (a cross-family `direct` / `converse` / `gpt-oss` target) keeps `bedrock-runtime` and the verbatim `endpoint_assumption`, and discloses the -unhonored selector in `rationale` and `capability_gaps[]`. A matched selector is validated against the API it requires before the path is chosen (`web_search` and `background_inference` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. Every other row keeps +unhonored selector in `rationale` and `capability_gaps[]`. A matched selector is validated against the API it requires before the path is chosen (`web_search`, `background_inference`, and `server_side_tools` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. Every other row keeps `bedrock-runtime` and records `endpoint_assumption` — string, REQUIRED on every non-`mantle_*` row, exactly: `No Mantle-only requirement was recorded for ; bedrock-runtime selected. If this workload needs background inference, server-side tools, Projects/Workspaces, or diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index dce73fd38..76a573a4f 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -357,7 +357,7 @@ limited to: `bedrock-mantle:CreateInference`, same page, read 2026-11-09) and gets no `bedrock:GetInvoke`. Verify with create-plus-retrieve: `r = client.responses.create(...)` then `client.responses.retrieve(r.id)` under the attached role. -6. Web Search → a row with `mantle_selector: web_search` needs `bedrock-websearch:InvokeSearch` and +6. Web Search → a confirmed `web_search` requirement (the row's `mantle_selector`, or any other `detected`/`user` entry in `ai_constraints.mantle_requirements`, including when the primary selector is `background_inference`) needs `bedrock-websearch:InvokeSearch` and `bedrock-websearch:InvokeFetch` on `*` (`iam_policy.py --mantle-selector web_search` emits them with an `aws:RequestedRegion` condition); the client sets `"external_web_access": false` on the `web_search` tool unless the user explicitly confirmed external access, in which case `--external-web-access` also grants diff --git a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md index 74a9144d5..22f59eb54 100644 --- a/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/azure-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -243,11 +243,11 @@ workload: `mantle_openai_chat` for a Chat Completions source, `mantle_messages` for an Anthropic SDK source — with `mantle_selector` = that selector. Several matching entries → the first in the policy-table order `background_inference, server_side_tools, web_search, projects, workspaces, - in_region_processing` becomes `mantle_selector`; name the others in the row's `rationale`. + in_region_processing` becomes `mantle_selector`; name the others in the row's `rationale`. Generate and the report pass every matched selector to `iam_policy.py --mantle-selector`; a grant such as Web Search is not implied by the primary selector. `in_region_processing` applies only when `bedrock-runtime` serves the model in the entry's `region` through a CRIS profile alone (the CRIS-only gate in `ai-openai-to-bedrock.md`); otherwise the workload stays on `bedrock-runtime` in that Region. -- A matched selector is validated against the API it requires before the path is chosen (`web_search` and `background_inference` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. +- A matched selector is validated against the API it requires before the path is chosen (`web_search`, `background_inference`, and `server_side_tools` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. - A matching entry on a workload whose source API has no Mantle path (Gemini / Vertex AI, or a workload whose target is a cross-family `direct` / `converse` / `gpt-oss` row) does not change the endpoint: the row keeps `bedrock-runtime`, records `endpoint_assumption` verbatim (below), diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md index 160f0ba14..5640e583a 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/design/design-ai.md @@ -112,8 +112,8 @@ Read `preferences.json` → `ai_constraints.mantle_requirements` (written by Cla **Derivation rule — apply per workload (`workloads[]` from `preferences.json`, falling back to the profile; when the profile has no `workloads[]`, every entry is `*` and applies to every row):** 1. Collect the `mantle_requirements.value[]` entries whose `workload_id` equals the workload's id or is `*`, and whose `selector` is one of `background_inference | server_side_tools | web_search | projects | workspaces | in_region_processing` with `source` ∈ `detected | user`. Entries with `selector: "none"` or `source: "none"` match nothing. -1b. **Validate the matched selector against the API it requires.** `web_search` and `background_inference` require the Responses API (`web-search.html`; `inference-responses-api.html` — read 2026-11-09); `workspaces` requires the Anthropic Messages API (`anthropic-workspace-id`); `projects` requires an OpenAI-compatible API (`OpenAI-Project`). A selector whose provider does not match the workload's source API — `workspaces` on an OpenAI SDK workload, `projects` on an Anthropic SDK workload — is a mismatch: the row keeps `bedrock-runtime`, records `endpoint_assumption`, discloses the mismatch in `rationale` and `capability_gaps[]`, and Clarify is re-run; never invent a Mantle path. A Chat Completions source (or an Anthropic Messages source) matched by a Responses-only selector (`web_search`, `background_inference`) is an **explicit API-continuity decision** recorded in the plan and the Present Summary, never a silent same-API Mantle row: (i) keep the source API on `bedrock-runtime` and drop the requirement — the row stays `runtime_openai_cris` / `runtime_messages` with `endpoint_assumption` and a `capability_gaps[]` entry naming the dropped selector and the decision; or (ii) move to `mantle_openai_responses` with that `mantle_selector`, `mantle_requirement`, and the Chat → Responses reshape delta in Generate. `mantle_openai_chat` / `mantle_messages` is never recorded for a Responses-only selector. -2. **Any match that passes 1b → the Mantle path for the workload's source API:** `"mantle_openai_responses"` for a Responses source, `"mantle_openai_chat"` for a Chat Completions source, `"mantle_messages"` for an Anthropic SDK source. Set `bedrock_models[].mantle_selector` to the matched selector; when several entries match, take the first in the policy-table order `background_inference, server_side_tools, web_search, projects, workspaces, in_region_processing` and name the others in the row's `rationale`. Copy the matched entry into `bedrock_models[].mantle_requirement` — `{ workload_id, selector, source, evidence?, region? }` where `evidence` is present when `source` is `detected` (the Clarify entry's `file:line construct`) and `region` is present iff `selector` is `in_region_processing`. +1b. **Validate the matched selector against the API it requires.** `web_search`, `background_inference`, and `server_side_tools` require the Responses API (`web-search.html`; `inference-responses-api.html`; `tool-use-server-side.html` — read 2026-11-09); `workspaces` requires the Anthropic Messages API (`anthropic-workspace-id`); `projects` requires an OpenAI-compatible API (`OpenAI-Project`). A selector whose provider does not match the workload's source API — `workspaces` on an OpenAI SDK workload, `projects` on an Anthropic SDK workload — is a mismatch: the row keeps `bedrock-runtime`, records `endpoint_assumption`, discloses the mismatch in `rationale` and `capability_gaps[]`, and Clarify is re-run; never invent a Mantle path. A Chat Completions source (or an Anthropic Messages source) matched by a Responses-only selector (`web_search`, `background_inference`, `server_side_tools`) is an **explicit API-continuity decision** recorded in the plan and the Present Summary, never a silent same-API Mantle row: (i) keep the source API on `bedrock-runtime` and drop the requirement — the row stays `runtime_openai_cris` / `runtime_messages` with `endpoint_assumption` and a `capability_gaps[]` entry naming the dropped selector and the decision; or (ii) move to `mantle_openai_responses` with that `mantle_selector`, `mantle_requirement`, and the Chat → Responses reshape delta in Generate. `mantle_openai_chat` / `mantle_messages` is never recorded for a Responses-only selector. +2. **Any match that passes 1b → the Mantle path for the workload's source API:** `"mantle_openai_responses"` for a Responses source, `"mantle_openai_chat"` for a Chat Completions source, `"mantle_messages"` for an Anthropic SDK source. Set `bedrock_models[].mantle_selector` to the matched selector; when several entries match, take the first in the policy-table order `background_inference, server_side_tools, web_search, projects, workspaces, in_region_processing` and name the others in the row's `rationale`. Generate and the report pass every matched selector to `iam_policy.py --mantle-selector`; a grant such as Web Search is not implied by the primary selector. Copy the matched entry into `bedrock_models[].mantle_requirement` — `{ workload_id, selector, source, evidence?, region? }` where `evidence` is present when `source` is `detected` (the Clarify entry's `file:line construct`) and `region` is present iff `selector` is `in_region_processing`. - `in_region_processing` still obeys the existing CRIS-only gate in `references/vendored/ai/ai-openai-to-bedrock.md`: it selects Mantle only when `bedrock-runtime` serves the selected model in the entry's `region` through a CRIS profile alone; when a runtime in-Region form exists, the workload stays on `bedrock-runtime` in that Region and the row records the `endpoint_assumption` below with a `rationale` note that the Region need is met on runtime. - **A match on a workload whose source API has no Mantle path** (Gemini / Vertex AI via `google-genai`, or any workload whose target is a cross-family `"direct"` / `"converse"` / `"gpt-oss"` row) does not change the endpoint. The row keeps its `bedrock-runtime` path and records the `endpoint_assumption` sentence below verbatim (it is the fixed runtime-lane disclosure), and the recorded requirement is disclosed as a capability gap: name the unhonored selector with its `source` and `evidence` in the row's `rationale` and add a `capability_gaps[]` entry (`" requested in Clarify (, ) — no bedrock-mantle path for this source API"`); for `in_region_processing` add a `regional_warnings[]` entry instead. Never invent a Mantle path for that source and never drop the answer silently; the Present Summary endpoint line names the gap. 3. **`mantle_only_model_or_api` is never a Clarify value.** When the selected model or API exists only on `bedrock-mantle` per the catalog (`references/vendored/ai/ai-openai-to-bedrock.md` / `references/shared/openai-on-bedrock.md` access-path tables), record the Mantle path with `mantle_selector: "mantle_only_model_or_api"` and `mantle_requirement: { "workload_id": "", "selector": "mantle_only_model_or_api", "source": "catalog", "evidence": "" }`. diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md index 6049f6ede..e95a2ec8f 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/phases/generate/generate-artifacts-ai.md @@ -216,7 +216,7 @@ Generate `ai-migration/setup_bedrock.sh`. - Dry-run by default (`--execute` flag to run for real) - Step 1 — Request model access: List each model from `aws-design-ai.json` → `bedrock_models[].aws_model_id` and the embedding model -- Step 2 — Create IAM role: Trust policy for the compute platform (Lambda, ECS, or EC2 based on `aws-design.json` if present). Match inference permissions to the selected endpoint. Mantle needs `bedrock-mantle:CreateInference` scoped to `arn:aws:bedrock-mantle:{region}:{account_id}:project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` when using bearer credentials. Runtime needs `bedrock:InvokeModel` / `bedrock:InvokeModelWithResponseStream` against the selected foundation-model ARN and, for CRIS, the exact inference-profile ARN. For EVERY `runtime_openai_cris` row whose `required_client_api` is `responses` or `chat_completions` — classified by id form, not model family: any `us.`/`global.` (and, for GPT-5.6 Terra and Luna, `in.`) `openai.gpt-*` CRIS id, so GPT-5.6 Sol / Terra / Luna, the GPT-6 family, GPT-5.5 and GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default`: a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, "Permissions", read 2026-11-07), and each of those model cards (read 2026-11-07) repeats the requirement for its runtime CRIS ids. `iam_policy.py --target =` (one pair per `bedrock_models[]` row, each row's own API, so a GPT Chat / Responses secondary beside a Claude Messages primary still gets it) emits that third ARN for exactly those rows. A row with `required_client_api: "converse"` (the Converse-only application-inference-profile route) and every Anthropic / Nova / gpt-oss id get no such grant; a bare `openai.gpt-5.5` / `openai.gpt-5.4` / `openai.gpt-5.6-*` / `openai.gpt-6-*` id is a mantle target and gets no such grant. Runtime actions do not authorize Mantle. **Retained stored-response operations:** when the source code keeps `client.responses.retrieve(id)` (or `.cancel(id)` / `.delete(id)`; scan for `responses.retrieve(` / `responses.cancel(` / `responses.delete(`), a runtime `responses` row ALSO needs `bedrock:GetInvoke` (and `bedrock:CancelInvoke` / `bedrock:DeleteInvoke` only for the ops actually retained) on the same `arn:aws:bedrock:{region}:{account}:project/default` — creating a response authorizes `bedrock:InvokeModel` on the target and the project, while retrieving, canceling or deleting a stored response authorizes those actions on the project (same page, "Permissions", read 2026-11-08); without them creation and the ping pass and the app's existing retrieve is denied. `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits that `BedrockStoredResponseOps` statement for a runtime `responses` row only and refuses (`reason: stored_response_ops_no_responses_row`) for a `converse`, `messages` or `chat_completions` plan, which has no stored responses on either endpoint. A bare Mantle id on `responses` (a `mantle_openai_responses` row) satisfies retained ops through the `BedrockMantleInference` statement: stored responses are retrievable on Mantle at `GET /v1/responses/{id}` under `bedrock-mantle:CreateInference` (same page, "Permissions", read 2026-11-09), so no `bedrock:GetInvoke` is added and the helper does not refuse. Step 5's verification must then also retrieve the response it just created. **Web Search grants:** when a `bedrock_models[]` row has `mantle_selector: web_search`, pass `iam_policy.py --mantle-selector web_search` and the policy gains a `BedrockWebSearch` statement — `bedrock-websearch:InvokeSearch` and `bedrock-websearch:InvokeFetch` on resource `*` with an `aws:RequestedRegion` condition (all three `bedrock-websearch` actions take `*`; that is the service's only condition key — , read 2026-11-09). The generated client sets `"external_web_access": false` on the `web_search` tool by default: the parameter defaults to `true`, and without `bedrock-websearch:ExternalWebAccess` every Fetch then fails authorization before the cache is read while the request still returns 200 (, "Controlling external web access", read 2026-11-09). Only when the user explicitly confirmed external web access in Clarify, pass `--external-web-access` as well (adds `bedrock-websearch:ExternalWebAccess`) and leave the parameter `true`. Web Search is served on `bedrock-mantle` Responses only, so the helper refuses (`reason: web_search_without_mantle_row`) when no row is a bare Mantle id; a plan without a `web_search` row gets no `bedrock-websearch:*` action. +- Step 2 — Create IAM role: Trust policy for the compute platform (Lambda, ECS, or EC2 based on `aws-design.json` if present). Match inference permissions to the selected endpoint. Mantle needs `bedrock-mantle:CreateInference` scoped to `arn:aws:bedrock-mantle:{region}:{account_id}:project/*`, plus `bedrock-mantle:CallWithBearerToken` on `*` when using bearer credentials. Runtime needs `bedrock:InvokeModel` / `bedrock:InvokeModelWithResponseStream` against the selected foundation-model ARN and, for CRIS, the exact inference-profile ARN. For EVERY `runtime_openai_cris` row whose `required_client_api` is `responses` or `chat_completions` — classified by id form, not model family: any `us.`/`global.` (and, for GPT-5.6 Terra and Luna, `in.`) `openai.gpt-*` CRIS id, so GPT-5.6 Sol / Terra / Luna, the GPT-6 family, GPT-5.5 and GPT-5.4 (`us.openai.gpt-5.5` / `global.openai.gpt-5.5`, `us.openai.gpt-5.4` / `global.openai.gpt-5.4`), or their `inference-profile/` ARNs — also grant `bedrock:InvokeModel` on `arn:aws:bedrock:{region}:{account}:project/default`: a Responses request on `bedrock-runtime` authorizes the default project separately from the inference target (, "Permissions", read 2026-11-07), and each of those model cards (read 2026-11-07) repeats the requirement for its runtime CRIS ids. `iam_policy.py --target =` (one pair per `bedrock_models[]` row, each row's own API, so a GPT Chat / Responses secondary beside a Claude Messages primary still gets it) emits that third ARN for exactly those rows. A row with `required_client_api: "converse"` (the Converse-only application-inference-profile route) and every Anthropic / Nova / gpt-oss id get no such grant; a bare `openai.gpt-5.5` / `openai.gpt-5.4` / `openai.gpt-5.6-*` / `openai.gpt-6-*` id is a mantle target and gets no such grant. Runtime actions do not authorize Mantle. **Retained stored-response operations:** when the source code keeps `client.responses.retrieve(id)` (or `.cancel(id)` / `.delete(id)`; scan for `responses.retrieve(` / `responses.cancel(` / `responses.delete(`), a runtime `responses` row ALSO needs `bedrock:GetInvoke` (and `bedrock:CancelInvoke` / `bedrock:DeleteInvoke` only for the ops actually retained) on the same `arn:aws:bedrock:{region}:{account}:project/default` — creating a response authorizes `bedrock:InvokeModel` on the target and the project, while retrieving, canceling or deleting a stored response authorizes those actions on the project (same page, "Permissions", read 2026-11-08); without them creation and the ping pass and the app's existing retrieve is denied. `iam_policy.py --stored-response-ops retrieve[,cancel,delete]` emits that `BedrockStoredResponseOps` statement for a runtime `responses` row only and refuses (`reason: stored_response_ops_no_responses_row`) for a `converse`, `messages` or `chat_completions` plan, which has no stored responses on either endpoint. A bare Mantle id on `responses` (a `mantle_openai_responses` row) satisfies retained ops through the `BedrockMantleInference` statement: stored responses are retrievable on Mantle at `GET /v1/responses/{id}` under `bedrock-mantle:CreateInference` (same page, "Permissions", read 2026-11-09), so no `bedrock:GetInvoke` is added and the helper does not refuse. Step 5's verification must then also retrieve the response it just created. **Web Search grants:** when `web_search` is a confirmed mantle requirement (the row's `mantle_selector`, or any other `detected`/`user` entry in `ai_constraints.mantle_requirements`, including when the primary selector is `background_inference`), pass `iam_policy.py --mantle-selector web_search` and the policy gains a `BedrockWebSearch` statement — `bedrock-websearch:InvokeSearch` and `bedrock-websearch:InvokeFetch` on resource `*` with an `aws:RequestedRegion` condition (all three `bedrock-websearch` actions take `*`; that is the service's only condition key — , read 2026-11-09). The generated client sets `"external_web_access": false` on the `web_search` tool by default: the parameter defaults to `true`, and without `bedrock-websearch:ExternalWebAccess` every Fetch then fails authorization before the cache is read while the request still returns 200 (, "Controlling external web access", read 2026-11-09). Only when the user explicitly confirmed external web access in Clarify, pass `--external-web-access` as well (adds `bedrock-websearch:ExternalWebAccess`) and leave the parameter `true`. Web Search is served on `bedrock-mantle` Responses only, so the helper refuses (`reason: web_search_without_mantle_row`) when no row is a bare Mantle id; a plan without a `web_search` row gets no `bedrock-websearch:*` action. - If the runtime client uses `BEDROCK_INFERENCE_PROFILE_ARN` from the generated monitoring output, also authorize that exact `application-inference-profile/…` ARN, retaining the backing model/CRIS resources needed for the selected model and fallback. Resolve it from the same output used by the client; do not grant every application profile. Re-run setup after enabling or changing this override so the effective invocation target and IAM remain aligned. See [inference-profile IAM prerequisites](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-prereq.html). - Compliance carry-through: if `preferences.json` → `design_constraints.compliance` is set and not `none`/`unknown` (full-flow Q2 / AI-only Q1.5), the script and its printed notes MUST reflect the design's Step 0.7 constraints — `AWS_REGION` from the compliance-constrained region (GovCloud for fedramp, EU for gdpr), `eu.` inference-profile model IDs for gdpr, and for hipaa a printed warning that Bedrock invocation logs retain original content (KMS-encrypt + restrict IAM on the log group before enabling) - Step 3 — Print required environment variables: `AWS_REGION`, `AI_PROVIDER=bedrock`, model IDs diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md index a475aac93..d6b223f4a 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/shared/schema-design-aws-ai.md @@ -94,7 +94,7 @@ same vocabulary as the shared contract (`references/vendored/ai/schema-design-aw - `mantle_openai_responses` / `mantle_openai_chat` / `mantle_messages` — the `bedrock-mantle` endpoint; each such row MUST carry `mantle_selector`, one of `background_inference | server_side_tools | web_search | projects | workspaces | mantle_only_model_or_api | - in_region_processing`. API continuity alone is not a selector. A matched selector is validated against the API it requires before the path is chosen (`web_search` and `background_inference` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. + in_region_processing`. API continuity alone is not a selector. A matched selector is validated against the API it requires before the path is chosen (`web_search`, `background_inference`, and `server_side_tools` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. - `bedrock_models[].mantle_requirement` — object, REQUIRED on every `mantle_*` row, absent otherwise: `{ workload_id, selector, source, evidence?, region? }` — `selector` equals the row's `mantle_selector`; `source` ∈ `detected | user | catalog` (`detected`/`user` entries are copied diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md index dce73fd38..76a573a4f 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/ai-openai-to-bedrock.md @@ -357,7 +357,7 @@ limited to: `bedrock-mantle:CreateInference`, same page, read 2026-11-09) and gets no `bedrock:GetInvoke`. Verify with create-plus-retrieve: `r = client.responses.create(...)` then `client.responses.retrieve(r.id)` under the attached role. -6. Web Search → a row with `mantle_selector: web_search` needs `bedrock-websearch:InvokeSearch` and +6. Web Search → a confirmed `web_search` requirement (the row's `mantle_selector`, or any other `detected`/`user` entry in `ai_constraints.mantle_requirements`, including when the primary selector is `background_inference`) needs `bedrock-websearch:InvokeSearch` and `bedrock-websearch:InvokeFetch` on `*` (`iam_policy.py --mantle-selector web_search` emits them with an `aws:RequestedRegion` condition); the client sets `"external_web_access": false` on the `web_search` tool unless the user explicitly confirmed external access, in which case `--external-web-access` also grants diff --git a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md index 74a9144d5..22f59eb54 100644 --- a/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/gcp-to-aws/references/vendored/ai/schema-design-aws-ai.md @@ -243,11 +243,11 @@ workload: `mantle_openai_chat` for a Chat Completions source, `mantle_messages` for an Anthropic SDK source — with `mantle_selector` = that selector. Several matching entries → the first in the policy-table order `background_inference, server_side_tools, web_search, projects, workspaces, - in_region_processing` becomes `mantle_selector`; name the others in the row's `rationale`. + in_region_processing` becomes `mantle_selector`; name the others in the row's `rationale`. Generate and the report pass every matched selector to `iam_policy.py --mantle-selector`; a grant such as Web Search is not implied by the primary selector. `in_region_processing` applies only when `bedrock-runtime` serves the model in the entry's `region` through a CRIS profile alone (the CRIS-only gate in `ai-openai-to-bedrock.md`); otherwise the workload stays on `bedrock-runtime` in that Region. -- A matched selector is validated against the API it requires before the path is chosen (`web_search` and `background_inference` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. +- A matched selector is validated against the API it requires before the path is chosen (`web_search`, `background_inference`, and `server_side_tools` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. - A matching entry on a workload whose source API has no Mantle path (Gemini / Vertex AI, or a workload whose target is a cross-family `direct` / `converse` / `gpt-oss` row) does not change the endpoint: the row keeps `bedrock-runtime`, records `endpoint_assumption` verbatim (below), diff --git a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md index dce73fd38..76a573a4f 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/ai-openai-to-bedrock.md @@ -357,7 +357,7 @@ limited to: `bedrock-mantle:CreateInference`, same page, read 2026-11-09) and gets no `bedrock:GetInvoke`. Verify with create-plus-retrieve: `r = client.responses.create(...)` then `client.responses.retrieve(r.id)` under the attached role. -6. Web Search → a row with `mantle_selector: web_search` needs `bedrock-websearch:InvokeSearch` and +6. Web Search → a confirmed `web_search` requirement (the row's `mantle_selector`, or any other `detected`/`user` entry in `ai_constraints.mantle_requirements`, including when the primary selector is `background_inference`) needs `bedrock-websearch:InvokeSearch` and `bedrock-websearch:InvokeFetch` on `*` (`iam_policy.py --mantle-selector web_search` emits them with an `aws:RequestedRegion` condition); the client sets `"external_web_access": false` on the `web_search` tool unless the user explicitly confirmed external access, in which case `--external-web-access` also grants diff --git a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json index 1ba299cdb..933c76e11 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json +++ b/plugins/aws-startup-advisor/skills/shared/ai/models/openai-bedrock.json @@ -853,7 +853,8 @@ "model_id": "anthropic.claude-sonnet-5", "requires_cris": true, "guardrails_supported": true, - "evidence": "AWS model card (2026-07-21): Converse on bedrock-runtime checked; Geo (us/eu/au) and Global CRIS supported" + "prompt_caching_supported": true, + "evidence": "AWS model card (2026-07-21): Converse on bedrock-runtime checked; Geo (us/eu/au) and Global CRIS supported; prompt_caching_supported true: Claude explicit cache checkpoints (cachePoint) are supported on Converse (https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html, read 2026-11-09)" } } }, @@ -876,7 +877,8 @@ "model_id": "anthropic.claude-haiku-4-5-20251001-v1:0", "requires_cris": true, "guardrails_supported": true, - "evidence": "anthropic-bedrock.json catalog (verified 2026-09-24) runtime_converse entry (requires CRIS profile)" + "prompt_caching_supported": true, + "evidence": "anthropic-bedrock.json catalog (verified 2026-09-24) runtime_converse entry (requires CRIS profile); prompt_caching_supported true: Claude explicit cache checkpoints (cachePoint) are supported on Converse (https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html, read 2026-11-09)" } } }, @@ -970,7 +972,9 @@ "available": true, "model_id": "anthropic.claude-opus-5-5", "requires_cris": true, - "guardrails_supported": true + "guardrails_supported": true, + "prompt_caching_supported": true, + "evidence": "prompt_caching_supported true: Claude explicit cache checkpoints (cachePoint) are supported on Converse (https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html, read 2026-11-09)" } }, "generation": "bedrock_native", diff --git a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md index 74a9144d5..22f59eb54 100644 --- a/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md +++ b/plugins/aws-startup-advisor/skills/shared/ai/schema-design-aws-ai.md @@ -243,11 +243,11 @@ workload: `mantle_openai_chat` for a Chat Completions source, `mantle_messages` for an Anthropic SDK source — with `mantle_selector` = that selector. Several matching entries → the first in the policy-table order `background_inference, server_side_tools, web_search, projects, workspaces, - in_region_processing` becomes `mantle_selector`; name the others in the row's `rationale`. + in_region_processing` becomes `mantle_selector`; name the others in the row's `rationale`. Generate and the report pass every matched selector to `iam_policy.py --mantle-selector`; a grant such as Web Search is not implied by the primary selector. `in_region_processing` applies only when `bedrock-runtime` serves the model in the entry's `region` through a CRIS profile alone (the CRIS-only gate in `ai-openai-to-bedrock.md`); otherwise the workload stays on `bedrock-runtime` in that Region. -- A matched selector is validated against the API it requires before the path is chosen (`web_search` and `background_inference` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. +- A matched selector is validated against the API it requires before the path is chosen (`web_search`, `background_inference`, and `server_side_tools` require the Responses API; `workspaces` the Anthropic Messages API; `projects` an OpenAI-compatible API — read 2026-11-09): a Chat Completions or Anthropic Messages source matched by a Responses-only selector is an explicit API-continuity decision — keep the source API on `bedrock-runtime` and drop the requirement (disclosed in `capability_gaps[]`), or move to `mantle_openai_responses` — never a silent same-API Mantle row; a provider mismatch (`workspaces` on OpenAI, `projects` on Anthropic) keeps `bedrock-runtime` and re-runs Clarify. - A matching entry on a workload whose source API has no Mantle path (Gemini / Vertex AI, or a workload whose target is a cross-family `direct` / `converse` / `gpt-oss` row) does not change the endpoint: the row keeps `bedrock-runtime`, records `endpoint_assumption` verbatim (below), diff --git a/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py index 05ad3aeb2..31d30b188 100644 --- a/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py +++ b/plugins/aws-startup-advisor/tests/test_runtime_default_review_round3.py @@ -125,7 +125,7 @@ def test_report_recipe_names_the_web_search_grants_and_the_fetch_default() -> No def test_report_checklist_has_a_tool_level_permission_check() -> None: apply = _squash(_section(REPORT_GENERATOR.read_text(), "## How to Apply", "## How to Undo")) assert "Tool-level permission check" in apply - assert "Render ONLY when a `bedrock_models[]` row has `mantle_selector: web_search`" in apply + assert "Render ONLY when `web_search` is a confirmed mantle requirement" in apply assert 'tools=[{"type": "web_search", "external_web_access": False}]' in apply assert "`web_search_call` item or a `url_citation` annotation" in apply assert "CloudTrail Web Search data events for a denied `InvokeSearch` / `InvokeFetch`" in apply @@ -255,7 +255,7 @@ def test_step_08_validates_the_selector_against_its_api_before_choosing_a_path(p def test_schema_docs_carry_the_responses_only_selector_decision(path: Path) -> None: text = _squash(path.read_text()) assert "A matched selector is validated against the API it requires before the path is chosen" in text - assert "`web_search` and `background_inference` require the Responses API" in text + assert "`web_search`, `background_inference`, and `server_side_tools` require the Responses API" in text assert "explicit API-continuity decision" in text and "never a silent same-API Mantle row" in text assert "`workspaces` on OpenAI, `projects` on Anthropic" in text @@ -300,6 +300,6 @@ def test_design_fixture_corpus_never_records_a_responses_only_selector_on_a_same for row in json.loads(path.read_text())["ai_architecture"]["bedrock_models"]: selector = row.get("mantle_selector") if row.get("migration_path") == "mantle_openai_chat": - assert selector not in {"web_search", "background_inference"}, f"{path}: Responses-only selector on a Chat Mantle row" + assert selector not in {"web_search", "background_inference", "server_side_tools"}, f"{path}: Responses-only selector on a Chat Mantle row" if row.get("migration_path") == "mantle_messages": assert selector not in {"background_inference", "projects"}, f"{path}: OpenAI-only selector on a Messages Mantle row" From bfada0d1bec674e98163fd79c86324fd91677e57 Mon Sep 17 00:00:00 2001 From: Logan Kleier Date: Mon, 9 Nov 2026 14:49:29 -0800 Subject: [PATCH 41/41] fix(aws-startup-advisor): write the Mantle scan fixtures as schema 1.1 profiles The 1.1 scan contract requires workloads and integration. These two fixtures still said 1.0, so the fixture sweep failed after the main merge. Co-authored-by: Cursor --- .../ai-workload-profile.json | 33 ++++++++++++- .../ai-workload-profile.json | 47 ++++++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/ai-workload-profile.json b/plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/ai-workload-profile.json index 005e013c3..626a18305 100644 --- a/plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/ai-workload-profile.json +++ b/plugins/aws-startup-advisor/fixtures/bma-scan-anthropic-client-tools/ai-workload-profile.json @@ -1,5 +1,5 @@ { - "schema_version": "1.0", + "schema_version": "1.1", "ai_sources": [ { "provider": "anthropic", @@ -12,5 +12,34 @@ ] } ], - "gateways": [] + "gateways": [], + "workloads": [ + { + "workload_id": "anthropic_api.claude-sonnet-5.anthropic.messages.create.plain", + "provider": "anthropic", + "source_service": "anthropic_api", + "model": "claude-sonnet-5", + "sdk_method": "anthropic.messages.create", + "capability": "text_generation", + "capability_confidence": "high", + "structured_output": false, + "call_sites": [{"file": "app/agent.py", "line": 7}] + } + ], + "integration": { + "pattern": "direct_sdk", + "languages": ["python"], + "frameworks": [], + "capabilities_summary": { + "text_generation": true, + "streaming": false, + "function_calling": true, + "vision": false, + "embeddings": false, + "batch_processing": false, + "speech_to_text": false, + "text_to_speech": false, + "image_generation": false + } + } } diff --git a/plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/ai-workload-profile.json b/plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/ai-workload-profile.json index b4d19af81..00eab674d 100644 --- a/plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/ai-workload-profile.json +++ b/plugins/aws-startup-advisor/fixtures/bma-scan-openai-mantle-signals/ai-workload-profile.json @@ -1,5 +1,5 @@ { - "schema_version": "1.0", + "schema_version": "1.1", "ai_sources": [ { "provider": "openai", @@ -13,6 +13,51 @@ } ], "gateways": [], + "workloads": [ + { + "workload_id": "openai_api.gpt-4o.openai.chat.completions.create.plain", + "provider": "openai", + "source_service": "openai_api", + "model": "gpt-4o", + "sdk_method": "openai.chat.completions.create", + "capability": "text_generation", + "capability_confidence": "medium", + "structured_output": false, + "call_sites": [{"file": "app/llm.py", "line": 28}] + }, + { + "workload_id": "openai_api.gpt-4o.openai.responses.create.plain", + "provider": "openai", + "source_service": "openai_api", + "model": "gpt-4o", + "sdk_method": "openai.responses.create", + "capability": "text_generation", + "capability_confidence": "high", + "structured_output": false, + "call_sites": [ + {"file": "app/llm.py", "line": 9}, + {"file": "app/llm.py", "line": 14}, + {"file": "app/llm.py", "line": 19}, + {"file": "app/llm.py", "line": 37} + ] + } + ], + "integration": { + "pattern": "direct_sdk", + "languages": ["python"], + "frameworks": [], + "capabilities_summary": { + "text_generation": true, + "streaming": false, + "function_calling": true, + "vision": false, + "embeddings": false, + "batch_processing": false, + "speech_to_text": false, + "text_to_speech": false, + "image_generation": false + } + }, "mantle_signals": [ {"selector": "background_inference", "provider": "openai", "source_service": "openai_api", "model": "gpt-4o", "evidence": [{"provenance": "code", "confidence": "high", "detail": "app/llm.py:9 background=True"}]}, {"selector": "web_search", "provider": "openai", "source_service": "openai_api", "model": "gpt-4o", "evidence": [{"provenance": "code", "confidence": "high", "detail": "app/llm.py:14 \"type\": \"web_search\""}]},