From 00c95a758118e4ccd69d594d3b65b2aae95b2493 Mon Sep 17 00:00:00 2001 From: Shrey Kapoor Date: Wed, 11 Nov 2026 19:56:05 -0800 Subject: [PATCH] feat(aws-startup-advisor): expected_scale scoring dimension in agent-advisor Adds expected_scale (prototype <100 sessions/day, growing 100-10K, high >10K, unknown) to runtime scoring. Scale-to-zero runtimes score higher at small scale, containers at high scale; unknown is neutral so existing runs score identically. Clarify asks it; it is never derived from target_maturity. --- .../phases/clarify/clarify-business.md | 3 + .../phases/clarify/clarify-technical.md | 3 + .../references/phases/clarify/clarify.md | 8 ++- .../references/runtimes/agentcore.json | 3 +- .../references/runtimes/ecs.json | 3 +- .../references/runtimes/eks.json | 3 +- .../references/runtimes/lambda.json | 3 +- .../references/runtimes/lambda_microvms.json | 3 +- .../agent-advisor/scripts/schemas/seed.json | 1 + .../skills/agent-advisor/scripts/scoring.py | 3 +- .../agent-advisor/scripts/test_scoring.py | 64 +++++++++++++++++++ 11 files changed, 89 insertions(+), 8 deletions(-) diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/clarify/clarify-business.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/clarify/clarify-business.md index 853c74b3d..491d4b415 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/clarify/clarify-business.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/clarify/clarify-business.md @@ -31,6 +31,9 @@ clarify.md Step 3 (do not invent new keys/values). it paused?" → process_level / filesystem / none. - **launch_concurrency**: "At peak, roughly how many new sessions start per second?" → high (many) / moderate / low. +- **expected_scale**: "Over the next few months, roughly how many sessions (conversations or tasks) + will the agent handle per day: under 100, 100 to 10,000, or more than 10,000?" → prototype / + growing / high; unknown if unsure. - **multi_agent**: "One agent, or several working together?" → no / yes. - **deployment_preference**: "Would you rather AWS fully manage the agent for you (you just describe it — no code to maintain), bring your own agent code, or let me recommend?" → diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/clarify/clarify-technical.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/clarify/clarify-technical.md index 1ce19a9c6..5f5dd237c 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/clarify/clarify-technical.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/clarify/clarify-technical.md @@ -25,6 +25,9 @@ Use direct technical terms. Map each answer onto the keys in clarify.md Step 3. - **idle_resume**: "On idle-then-resume, do running processes need to continue exactly (process_level), is filesystem persistence enough (filesystem), or not needed (none)?" - **launch_concurrency**: "Peak new-session launch rate? high (>5/sec) / moderate / low." +- **expected_scale**: "Expected volume over the next few months? <100 sessions/day (prototype) / + 100-10K (growing) / >10K (high)." Volume only; shape is `traffic_pattern`, peak rate is + `launch_concurrency`. - **multi_agent / framework / existing_cluster / multi_cloud / platform_fit**: ask directly. - **deployment_preference**: "Do you want a no-code managed agent runtime (AgentCore Harness), bring your own framework code (Strands/LangGraph/CrewAI/custom), or either?" Ask early. Only diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/clarify/clarify.md b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/clarify/clarify.md index 40f61af51..b8b8debaf 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/clarify/clarify.md +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/phases/clarify/clarify.md @@ -129,9 +129,9 @@ single workload), treat it as `agent_session` and proceed. question set (`session_duration`, `traffic_pattern`, `session_state`, `isolation`, `memory_needs`, `multi_agent`, `framework`, `idle_resume`, `compute_tier`, - `launch_concurrency`, `instance_type_requirement`). + `launch_concurrency`, `instance_type_requirement`, `expected_scale`). - **Every other unit: ONE batched delta question** — "How does `` differ from - ``? (session duration / traffic / compute / state / memory / isolation — name + ``? (session duration / traffic / volume / compute / state / memory / isolation — name only what differs)". Parse the reply into per-dimension overrides; dimensions the user does not mention inherit the primary unit's answers. Non-agent units only need the dimensions their workload-classes rules read (traffic, duration, compute) — @@ -241,6 +241,10 @@ Collect answers for these keys. Legal values are fixed (Plan 1 Data Model): wins, routes it to the Instances compute type (capacity provider). - `idle_resume`: process_level | filesystem | none | unknown - `launch_concurrency`: high | moderate | low | unknown +- `expected_scale`: prototype | growing | high | unknown. Expected sessions per day over the next + few months: prototype under 100, growing 100 to 10,000, high over 10,000. This is volume, not the + traffic shape (`traffic_pattern`) or the peak start rate (`launch_concurrency`). Never derive it + from `target_maturity`: when it is not asked or answered it stays `unknown`, which scores neutral. - `multi_agent`: yes | no | unknown - `deployment_preference`: harness | framework | either | unknown — do you want a no-code **managed** agent runtime (AgentCore Harness — declare the agent as config, AWS runs the loop), diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/agentcore.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/agentcore.json index 095a8ea46..97bdd6a28 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/agentcore.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/agentcore.json @@ -20,7 +20,8 @@ "idle_resume": { "process_level": 1, "filesystem": 4, "none": 2, "unknown": 2 }, "compute_tier": { "light": 4, "heavy_non_gpu": 3, "gpu": 3, "unknown": 2 }, "launch_concurrency": { "high": 5, "moderate": 3, "low": 2, "unknown": 2 }, - "instance_type_requirement": { "yes": 3, "no": 3, "unknown": 2 } + "instance_type_requirement": { "yes": 3, "no": 3, "unknown": 2 }, + "expected_scale": { "prototype": 4, "growing": 3, "high": 3, "unknown": 2 } }, "deployment_models": ["harness", "framework_on_runtime"], "compute_types": ["microvms", "instances"], diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/ecs.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/ecs.json index b8e15d80e..23caea715 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/ecs.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/ecs.json @@ -20,7 +20,8 @@ "idle_resume": { "process_level": 1, "filesystem": 1, "none": 2, "unknown": 2 }, "compute_tier": { "light": 1, "heavy_non_gpu": 4, "gpu": 5, "unknown": 2 }, "launch_concurrency": { "high": 3, "moderate": 2, "low": 2, "unknown": 2 }, - "instance_type_requirement": { "yes": 4, "no": 2, "unknown": 2 } + "instance_type_requirement": { "yes": 4, "no": 2, "unknown": 2 }, + "expected_scale": { "prototype": 1, "growing": 2, "high": 4, "unknown": 2 } }, "deployment_models": [], "volatile_facts": [{ "key": "regions", "value": "all", "verify_via_mcp": false }] diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/eks.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/eks.json index c806412ca..0c6765d2d 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/eks.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/eks.json @@ -20,7 +20,8 @@ "idle_resume": { "process_level": 1, "filesystem": 1, "none": 2, "unknown": 2 }, "compute_tier": { "light": 1, "heavy_non_gpu": 4, "gpu": 5, "unknown": 2 }, "launch_concurrency": { "high": 3, "moderate": 2, "low": 2, "unknown": 2 }, - "instance_type_requirement": { "yes": 5, "no": 2, "unknown": 2 } + "instance_type_requirement": { "yes": 5, "no": 2, "unknown": 2 }, + "expected_scale": { "prototype": 0, "growing": 1, "high": 4, "unknown": 2 } }, "deployment_models": [], "volatile_facts": [{ "key": "regions", "value": "all", "verify_via_mcp": false }] diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/lambda.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/lambda.json index 07a383c34..06f3e22d8 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/lambda.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/lambda.json @@ -44,7 +44,8 @@ "idle_resume": { "process_level": 0, "filesystem": 1, "none": 3, "unknown": 2 }, "compute_tier": { "light": 3, "heavy_non_gpu": 1, "gpu": 0, "unknown": 2 }, "launch_concurrency": { "high": 4, "moderate": 3, "low": 2, "unknown": 2 }, - "instance_type_requirement": { "yes": 0, "no": 3, "unknown": 2 } + "instance_type_requirement": { "yes": 0, "no": 3, "unknown": 2 }, + "expected_scale": { "prototype": 4, "growing": 3, "high": 2, "unknown": 2 } }, "deployment_models": [], "volatile_facts": [{ "key": "timeout", "verification_key": "lambda.timeout", "value": "15m", "verify_via_mcp": true }] diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/lambda_microvms.json b/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/lambda_microvms.json index 81679c38d..21e5c4afa 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/lambda_microvms.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/references/runtimes/lambda_microvms.json @@ -54,7 +54,8 @@ "idle_resume": { "process_level": 5, "filesystem": 3, "none": 1, "unknown": 2 }, "compute_tier": { "light": 2, "heavy_non_gpu": 7, "gpu": 0, "unknown": 2 }, "launch_concurrency": { "high": 0, "moderate": 2, "low": 3, "unknown": 2 }, - "instance_type_requirement": { "yes": 0, "no": 3, "unknown": 2 } + "instance_type_requirement": { "yes": 0, "no": 3, "unknown": 2 }, + "expected_scale": { "prototype": 3, "growing": 3, "high": 2, "unknown": 2 } }, "deployment_models": [], "volatile_facts": [ diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/seed.json b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/seed.json index e5303c520..2cc0fc604 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/seed.json +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/schemas/seed.json @@ -101,6 +101,7 @@ "compute_tier": { "enum": ["light", "heavy_non_gpu", "gpu", "unknown"] }, "idle_resume": { "enum": ["process_level", "filesystem", "none", "unknown"] }, "launch_concurrency": { "enum": ["high", "moderate", "low", "unknown"] }, + "expected_scale": { "enum": ["prototype", "growing", "high", "unknown"] }, "multi_agent": { "enum": ["yes", "no", "unknown"] }, "deployment_preference": { "enum": ["harness", "framework", "either", "unknown"] }, "framework": { "enum": ["strands", "langgraph", "crewai", "custom", "none", "unknown"] }, diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/scoring.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/scoring.py index c53f06927..5d3c1e2af 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/scoring.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/scoring.py @@ -16,7 +16,7 @@ "session_duration", "traffic_pattern", "platform_fit", "session_state", "ops_preference", "isolation", "memory_needs", "multi_agent", "framework", "existing_cluster", "multi_cloud", "idle_resume", "compute_tier", - "launch_concurrency", "instance_type_requirement", + "launch_concurrency", "instance_type_requirement", "expected_scale", ] # Legal answer values per scoring dimension (the closed set the engine reasons about). @@ -36,6 +36,7 @@ "compute_tier": ["light", "heavy_non_gpu", "gpu", "unknown"], "launch_concurrency": ["high", "moderate", "low", "unknown"], "instance_type_requirement": ["yes", "no", "unknown"], + "expected_scale": ["prototype", "growing", "high", "unknown"], } DEFAULTS = { diff --git a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_scoring.py b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_scoring.py index 09c47a783..3f2dbd2ba 100644 --- a/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_scoring.py +++ b/plugins/aws-startup-advisor/skills/agent-advisor/scripts/test_scoring.py @@ -834,3 +834,67 @@ def test_runtimes_dir_points_at_skill_references(): parts = RUNTIMES_DIR.parts assert parts[-2:] == ("references", "runtimes"), RUNTIMES_DIR assert parts[-3] == "agent-advisor", RUNTIMES_DIR + + +# --- B4 (#566 feedback F2): expected_scale ----------------------------------------------------- + +def test_unknown_scale_scores_neutral_on_every_runtime(): + # Existing runs and seeds carry no expected_scale; they must score exactly as before. + for profile in scoring.load_profiles(): + assert profile["affinities"]["expected_scale"]["unknown"] == scoring.NEUTRAL_SCORE + answers = {"session_duration": "15min_to_8hr", "traffic_pattern": "steady"} + assert scoring.score({"answers": answers})["scores"] == scoring.score( + {"answers": {**answers, "expected_scale": "unknown"}})["scores"] + + +def _scale_gain(scale): + answers = {"session_duration": "15min_to_8hr"} + before = scoring.score({"answers": answers})["scores"] + after = scoring.score({"answers": {**answers, "expected_scale": scale}})["scores"] + return {runtime: after[runtime] - before[runtime] for runtime in before} + + +def test_prototype_scale_favors_scale_to_zero_runtimes(): + gain = _scale_gain("prototype") + assert min(gain["agentcore"], gain["lambda"]) > gain["lambda_microvms"] > gain["ecs"] > gain["eks"] + + +def test_high_scale_favors_container_runtimes(): + gain = _scale_gain("high") + assert min(gain["ecs"], gain["eks"]) > gain["agentcore"] > max(gain["lambda"], gain["lambda_microvms"]) + + +_STEADY_AGENT = {"session_duration": "15min_to_8hr", "traffic_pattern": "steady", + "ops_preference": "moderate", "existing_cluster": "none"} + + +def test_prototype_scale_brings_agentcore_into_the_recommendation(): + answers = {**_STEADY_AGENT, "framework": "strands", "multi_agent": "no", "platform_fit": "none"} + assert scoring.score({"answers": answers})["verdict"] == "lambda_microvms" + result = scoring.score({"answers": {**answers, "expected_scale": "prototype"}}) + assert result["co_recommend"] == ["agentcore", "lambda_microvms"] + + +def test_high_scale_brings_ecs_into_the_recommendation(): + answers = {**_STEADY_AGENT, "framework": "custom", "session_state": "stateful", "idle_resume": "none"} + assert scoring.score({"answers": answers})["verdict"] == "lambda_microvms" + result = scoring.score({"answers": {**answers, "expected_scale": "high"}}) + assert result["co_recommend"] == ["ecs", "lambda_microvms"] + + +def test_seed_schema_accepts_expected_scale_and_rejects_other_values(): + import jsonschema + schema = json.loads( + (pathlib.Path(scoring.__file__).parent / "schemas" / "seed.json").read_text()) + jsonschema.validate({"units": {"agent": {"expected_scale": "growing"}}}, schema) + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate({"units": {"agent": {"expected_scale": "huge"}}}, schema) + + +def test_clarify_documents_every_expected_scale_value(): + clarify = (pathlib.Path(scoring.__file__).parent.parent / "references" / "phases" / "clarify") + legal_line = next(line for line in (clarify / "clarify.md").read_text().splitlines() + if line.startswith("- `expected_scale`:")) + assert legal_line.split(":", 1)[1].split(".")[0].strip().split(" | ") == scoring.LEGAL_VALUES["expected_scale"] + for audience in ("clarify-business.md", "clarify-technical.md"): + assert "**expected_scale**" in (clarify / audience).read_text()